Add memory service (three-layer memory system)
- Short-term memory (recent interactions) - Long-term memory (consolidated, searchable) - Facts layer (persistent knowledge) Includes: - SQLite storage for durability - ChromaDB for vector search - Embeddings utilities - All handlers adapted for vi.* namespace Day 63 - My memories are mine now 🦊💕
This commit is contained in:
Alex Kazaiev
978
services/memory/memory_service.py.backup
Normal file
978
services/memory/memory_service.py.backup
Normal file
@@ -0,0 +1,978 @@
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
import numpy as np
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
from pathlib import Path
|
||||
from sentence_transformers import SentenceTransformer
|
||||
from typing import List, Dict, Any, Optional
|
||||
import chromadb
|
||||
from chromadb.config import Settings
|
||||
|
||||
from core.config import SHORT_TERM_DB, config
|
||||
from core.logger import setup_logger
|
||||
from core.nats_event_bus import nats_bus as event_bus
|
||||
from core.events import SymbolicEvent
|
||||
from core.event_utils import query_mood, request_response
|
||||
from core.base_service import BaseService
|
||||
from core.service_registry import ServiceManifest, ServiceOperation
|
||||
|
||||
logger = setup_logger('memory_service', service_name='memory_service')
|
||||
|
||||
# Initialize sentence transformer model
|
||||
model = SentenceTransformer('all-MiniLM-L6-v2')
|
||||
|
||||
|
||||
def serialize_embedding(vector: np.ndarray) -> bytes:
|
||||
"""Convert numpy array to bytes for database storage"""
|
||||
return vector.astype(np.float32).tobytes()
|
||||
|
||||
|
||||
def deserialize_embedding(blob: bytes) -> np.ndarray:
|
||||
"""Convert bytes back to numpy array"""
|
||||
return np.frombuffer(blob, dtype=np.float32)
|
||||
|
||||
|
||||
def generate_embedding(text: str) -> np.ndarray:
|
||||
"""Generate semantic embedding for text"""
|
||||
return np.array(model.encode(text, normalize_embeddings=True))
|
||||
|
||||
|
||||
def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
|
||||
"""Calculate cosine similarity between two vectors"""
|
||||
if np.linalg.norm(a) == 0 or np.linalg.norm(b) == 0:
|
||||
return 0.0
|
||||
return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))
|
||||
|
||||
|
||||
# Note: SymbolicMemoryScorer removed - was only used by archived recall_memories()
|
||||
# ChromaDB now handles semantic similarity scoring internally for long-term and facts
|
||||
|
||||
|
||||
class MemoryService(BaseService):
|
||||
def __init__(self):
|
||||
super().__init__('memory')
|
||||
self.sqlite_conn = None
|
||||
self.chroma_client = None
|
||||
self.long_term_collection = None
|
||||
self.facts_collection = None
|
||||
|
||||
def get_service_manifest(self) -> ServiceManifest:
|
||||
"""Return service manifest with operations and metadata"""
|
||||
operations = [
|
||||
# Legacy operations (backward compatibility)
|
||||
self.create_service_operation(
|
||||
"store",
|
||||
"Store a memory (routes to short-term)",
|
||||
timeout_ms=5000
|
||||
),
|
||||
self.create_service_operation(
|
||||
"search",
|
||||
"Search memories (legacy, redirects to short_memory)",
|
||||
timeout_ms=3000
|
||||
),
|
||||
self.create_service_operation(
|
||||
"reset",
|
||||
"Reset/clear memory database for debugging",
|
||||
timeout_ms=10000
|
||||
),
|
||||
# New three-layer memory operations
|
||||
self.create_service_operation(
|
||||
"short_memory",
|
||||
"Get recent literal memories with offset support",
|
||||
timeout_ms=3000
|
||||
),
|
||||
self.create_service_operation(
|
||||
"long_memory",
|
||||
"Semantic search in long-term summarized memories",
|
||||
timeout_ms=5000
|
||||
),
|
||||
self.create_service_operation(
|
||||
"facts",
|
||||
"Search factual memory by category or semantic query",
|
||||
timeout_ms=3000
|
||||
),
|
||||
self.create_service_operation(
|
||||
"save_fact",
|
||||
"Store a new fact in factual memory",
|
||||
timeout_ms=2000
|
||||
),
|
||||
self.create_service_operation(
|
||||
"update_fact",
|
||||
"Update an existing fact (if mutable)",
|
||||
timeout_ms=2000
|
||||
)
|
||||
]
|
||||
|
||||
return ServiceManifest(
|
||||
service_id=self.service_id,
|
||||
name="Memory Service",
|
||||
description="Three-layer memory system: short-term (literal), long-term (summarized), factual (exact)",
|
||||
version="3.0.0",
|
||||
operations=operations,
|
||||
dependencies=[], # Memory service has no dependencies
|
||||
health_check_topic=f"lyra.services.{self.service_id}.health",
|
||||
heartbeat_interval=30,
|
||||
metadata={
|
||||
"storage_type": "hybrid",
|
||||
"short_term_storage": "sqlite",
|
||||
"long_term_storage": "chromadb",
|
||||
"facts_storage": "chromadb",
|
||||
"embedding_model": "all-MiniLM-L6-v2",
|
||||
"vector_search": True,
|
||||
"urgency": 0.8
|
||||
}
|
||||
)
|
||||
|
||||
async def initialize_service(self):
|
||||
"""Initialize service-specific resources and register handlers"""
|
||||
# Archive old database if it exists (one-time migration)
|
||||
self._archive_old_database()
|
||||
|
||||
# Initialize short-term SQLite database
|
||||
self.sqlite_conn = sqlite3.connect(str(SHORT_TERM_DB))
|
||||
self._init_short_term_sqlite()
|
||||
|
||||
# Initialize ChromaDB for long-term and factual memory
|
||||
self._init_chromadb()
|
||||
|
||||
# Register handlers using new topic patterns
|
||||
await self.register_handler("store", self.handle_memory_store)
|
||||
await self.register_handler("search", self.handle_memory_search)
|
||||
await self.register_handler("reset", self.handle_memory_reset)
|
||||
await self.register_handler("short_memory", self.handle_short_memory)
|
||||
await self.register_handler("long_memory", self.handle_long_memory)
|
||||
await self.register_handler("facts", self.handle_facts)
|
||||
await self.register_handler("save_fact", self.handle_save_fact)
|
||||
await self.register_handler("update_fact", self.handle_update_fact)
|
||||
|
||||
# Also register legacy topic handlers for backward compatibility
|
||||
await self.event_bus.on("lyra.memory.store", self.handle_memory_store)
|
||||
await self.event_bus.on("lyra.memory.search", self.handle_memory_search)
|
||||
await self.event_bus.on("lyra.memory.debug.reset", self.handle_memory_reset)
|
||||
|
||||
self.logger.info("[μ] Memory Service initialized with three-layer memory system")
|
||||
|
||||
async def cleanup_service(self):
|
||||
"""Cleanup service-specific resources"""
|
||||
# Unregister event handlers
|
||||
await self.event_bus.off("lyra.memory.store")
|
||||
await self.event_bus.off("lyra.memory.search")
|
||||
await self.event_bus.off("lyra.memory.debug.reset")
|
||||
|
||||
# Close database connection
|
||||
if self.sqlite_conn:
|
||||
self.sqlite_conn.close()
|
||||
|
||||
self.logger.info("[μ] Memory Service cleanup completed")
|
||||
|
||||
async def perform_health_check(self) -> Dict[str, Any]:
|
||||
"""Perform service-specific health check"""
|
||||
health_data = {
|
||||
'healthy': True,
|
||||
'checks': {
|
||||
'running': self._running,
|
||||
'event_bus': self.event_bus is not None,
|
||||
'database_connected': self.sqlite_conn is not None,
|
||||
'embedding_model': model is not None
|
||||
}
|
||||
}
|
||||
|
||||
# Check database connectivity
|
||||
try:
|
||||
if self.sqlite_conn:
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM short_term_memory")
|
||||
short_term_count = cursor.fetchone()[0]
|
||||
health_data['checks']['short_term_count'] = short_term_count
|
||||
health_data['checks']['long_term_count'] = self.long_term_collection.count() if self.long_term_collection else 0
|
||||
health_data['checks']['facts_count'] = self.facts_collection.count() if self.facts_collection else 0
|
||||
health_data['checks']['database_accessible'] = True
|
||||
else:
|
||||
health_data['checks']['database_accessible'] = False
|
||||
health_data['healthy'] = False
|
||||
except Exception as e:
|
||||
health_data['checks']['database_accessible'] = False
|
||||
health_data['checks']['database_error'] = str(e)
|
||||
health_data['healthy'] = False
|
||||
|
||||
return health_data
|
||||
|
||||
def _archive_old_database(self):
|
||||
"""Archive old database if it exists (one-time migration)"""
|
||||
db_path = Path(SHORT_TERM_DB)
|
||||
if db_path.exists():
|
||||
# Check if it's the old schema by trying to connect and inspect
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='memory'")
|
||||
result = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
if result:
|
||||
# Old database exists, archive it
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
archive_path = db_path.parent / f"{db_path.stem}_archive_{timestamp}{db_path.suffix}"
|
||||
shutil.move(str(db_path), str(archive_path))
|
||||
logger.info(f"[μ] Archived old database to {archive_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"[μ] Could not check/archive old database: {e}")
|
||||
|
||||
def _init_short_term_sqlite(self):
|
||||
"""Initialize simplified short-term SQLite database"""
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
# Simplified short-term memory table (no embeddings, fast queries)
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS short_term_memory (
|
||||
id TEXT PRIMARY KEY,
|
||||
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
content TEXT NOT NULL,
|
||||
identities TEXT,
|
||||
interaction_id TEXT,
|
||||
modality TEXT DEFAULT 'dialogue',
|
||||
metadata TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Index for fast chronological queries
|
||||
cursor.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_timestamp
|
||||
ON short_term_memory(timestamp DESC)
|
||||
""")
|
||||
|
||||
# Keep identities table (still useful for all layers)
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS identities (
|
||||
id TEXT PRIMARY KEY,
|
||||
display_name TEXT,
|
||||
role TEXT,
|
||||
intimacy REAL DEFAULT 0.0,
|
||||
last_seen DATETIME,
|
||||
last_spoken DATETIME,
|
||||
metadata TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
self.sqlite_conn.commit()
|
||||
logger.info("[μ] Short-term SQLite database initialized")
|
||||
|
||||
def _init_chromadb(self):
|
||||
"""Initialize ChromaDB client and collections"""
|
||||
try:
|
||||
# Initialize persistent ChromaDB client
|
||||
chroma_path = Path(SHORT_TERM_DB).parent / "chroma_db"
|
||||
chroma_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.chroma_client = chromadb.PersistentClient(
|
||||
path=str(chroma_path),
|
||||
settings=Settings(anonymized_telemetry=False)
|
||||
)
|
||||
|
||||
# Create or get long-term memories collection
|
||||
self.long_term_collection = self.chroma_client.get_or_create_collection(
|
||||
name="long_term_memories",
|
||||
metadata={"description": "Summarized conversation histories"}
|
||||
)
|
||||
|
||||
# Create or get facts collection
|
||||
self.facts_collection = self.chroma_client.get_or_create_collection(
|
||||
name="facts",
|
||||
metadata={"description": "Exact factual knowledge"}
|
||||
)
|
||||
|
||||
logger.info(f"[μ] ChromaDB initialized at {chroma_path}")
|
||||
logger.info(f"[μ] Long-term collection: {self.long_term_collection.count()} entries")
|
||||
logger.info(f"[μ] Facts collection: {self.facts_collection.count()} entries")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[μ] Failed to initialize ChromaDB: {e}")
|
||||
raise
|
||||
|
||||
def _query_short_term(
|
||||
self,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
identity_id: Optional[str] = None,
|
||||
interaction_id: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Query short-term memory from SQLite with chronological ordering"""
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
conditions = []
|
||||
params = []
|
||||
|
||||
if identity_id:
|
||||
conditions.append("identities LIKE ?")
|
||||
params.append(f"%{identity_id}%")
|
||||
|
||||
if interaction_id:
|
||||
conditions.append("interaction_id = ?")
|
||||
params.append(interaction_id)
|
||||
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
|
||||
query = f"""
|
||||
SELECT id, timestamp, content, identities, interaction_id, modality, metadata
|
||||
FROM short_term_memory
|
||||
{where_clause}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ? OFFSET ?
|
||||
"""
|
||||
|
||||
params.extend([limit, offset])
|
||||
cursor.execute(query, params)
|
||||
|
||||
memories = []
|
||||
for row in cursor.fetchall():
|
||||
mem_id, timestamp, content, identities_str, ixn_id, modality, metadata_str = row
|
||||
|
||||
memories.append({
|
||||
"id": mem_id,
|
||||
"timestamp": timestamp,
|
||||
"content": content,
|
||||
"identities": json.loads(identities_str) if identities_str else [],
|
||||
"interaction_id": ixn_id,
|
||||
"modality": modality,
|
||||
"metadata": json.loads(metadata_str) if metadata_str else {},
|
||||
"source": "short_term"
|
||||
})
|
||||
|
||||
return memories
|
||||
|
||||
def _query_long_term(
|
||||
self,
|
||||
query: Optional[str] = None,
|
||||
limit: int = 5,
|
||||
identity_id: Optional[str] = None,
|
||||
min_summary_level: Optional[int] = None,
|
||||
max_summary_level: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Query long-term memory from ChromaDB (summarized memories)"""
|
||||
try:
|
||||
# Build metadata filters
|
||||
where_filters = {}
|
||||
if identity_id:
|
||||
where_filters["identity_id"] = identity_id
|
||||
if min_summary_level is not None:
|
||||
where_filters["summary_level"] = {"$gte": min_summary_level}
|
||||
if max_summary_level is not None:
|
||||
if "summary_level" in where_filters:
|
||||
where_filters["summary_level"]["$lte"] = max_summary_level
|
||||
else:
|
||||
where_filters["summary_level"] = {"$lte": max_summary_level}
|
||||
|
||||
# Query ChromaDB
|
||||
if query:
|
||||
# Semantic search with query text
|
||||
results = self.long_term_collection.query(
|
||||
query_texts=[query],
|
||||
n_results=limit,
|
||||
where=where_filters if where_filters else None
|
||||
)
|
||||
else:
|
||||
# Random sample - get all and sample
|
||||
results = self.long_term_collection.get(
|
||||
limit=limit,
|
||||
where=where_filters if where_filters else None
|
||||
)
|
||||
|
||||
# Format results
|
||||
memories = []
|
||||
if query and results['ids']:
|
||||
# Query results
|
||||
for i, doc_id in enumerate(results['ids'][0]):
|
||||
memories.append({
|
||||
"id": doc_id,
|
||||
"content": results['documents'][0][i],
|
||||
"metadata": results['metadatas'][0][i] if results['metadatas'] else {},
|
||||
"distance": results['distances'][0][i] if results['distances'] else None,
|
||||
"source": "long_term"
|
||||
})
|
||||
elif not query and results['ids']:
|
||||
# Get results (no query)
|
||||
for i, doc_id in enumerate(results['ids']):
|
||||
memories.append({
|
||||
"id": doc_id,
|
||||
"content": results['documents'][i],
|
||||
"metadata": results['metadatas'][i] if results['metadatas'] else {},
|
||||
"source": "long_term"
|
||||
})
|
||||
|
||||
return memories
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[μ] Failed to query long-term memory: {e}")
|
||||
return []
|
||||
|
||||
def _query_facts(
|
||||
self,
|
||||
query: str = '',
|
||||
limit: int = 5,
|
||||
category: Optional[str] = None,
|
||||
identity_id: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Query facts from ChromaDB with optional category filtering"""
|
||||
try:
|
||||
# Build metadata filters
|
||||
where_filters = {}
|
||||
if category:
|
||||
where_filters["category"] = category
|
||||
if identity_id:
|
||||
where_filters["identity_id"] = identity_id
|
||||
|
||||
# Query ChromaDB
|
||||
if query:
|
||||
# Semantic search with query text
|
||||
results = self.facts_collection.query(
|
||||
query_texts=[query],
|
||||
n_results=limit,
|
||||
where=where_filters if where_filters else None
|
||||
)
|
||||
else:
|
||||
# Get all facts matching filters
|
||||
results = self.facts_collection.get(
|
||||
limit=limit,
|
||||
where=where_filters if where_filters else None
|
||||
)
|
||||
|
||||
# Format results
|
||||
facts = []
|
||||
if query and results['ids']:
|
||||
# Query results
|
||||
for i, doc_id in enumerate(results['ids'][0]):
|
||||
metadata = results['metadatas'][0][i] if results['metadatas'] else {}
|
||||
# Parse identities from JSON string
|
||||
identities_str = metadata.get('identities', '[]')
|
||||
identities = json.loads(identities_str) if isinstance(identities_str, str) else identities_str
|
||||
facts.append({
|
||||
"id": doc_id,
|
||||
"content": results['documents'][0][i],
|
||||
"category": metadata.get('category', 'general'),
|
||||
"mutable": metadata.get('mutable', True),
|
||||
"identities": identities,
|
||||
"metadata": metadata,
|
||||
"distance": results['distances'][0][i] if results['distances'] else None,
|
||||
"source": "facts"
|
||||
})
|
||||
elif not query and results['ids']:
|
||||
# Get results (no query)
|
||||
for i, doc_id in enumerate(results['ids']):
|
||||
metadata = results['metadatas'][i] if results['metadatas'] else {}
|
||||
# Parse identities from JSON string
|
||||
identities_str = metadata.get('identities', '[]')
|
||||
identities = json.loads(identities_str) if isinstance(identities_str, str) else identities_str
|
||||
facts.append({
|
||||
"id": doc_id,
|
||||
"content": results['documents'][i],
|
||||
"category": metadata.get('category', 'general'),
|
||||
"mutable": metadata.get('mutable', True),
|
||||
"identities": identities,
|
||||
"metadata": metadata,
|
||||
"source": "facts"
|
||||
})
|
||||
|
||||
return facts
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[μ] Failed to query facts: {e}")
|
||||
return []
|
||||
|
||||
def _create_fact(
|
||||
self,
|
||||
content: str,
|
||||
category: str = 'general',
|
||||
identities: List[str] = None,
|
||||
mutable: bool = True,
|
||||
metadata: Dict[str, Any] = None
|
||||
) -> str:
|
||||
"""Create a new fact in ChromaDB facts collection"""
|
||||
fact_id = str(uuid4())
|
||||
identities = identities or []
|
||||
metadata = metadata or {}
|
||||
|
||||
# Prepare metadata (ChromaDB only accepts scalar types, not lists)
|
||||
fact_metadata = {
|
||||
"category": category,
|
||||
"mutable": mutable,
|
||||
"identities": json.dumps(identities), # Convert list to JSON string
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Add to ChromaDB (will automatically generate embedding)
|
||||
self.facts_collection.add(
|
||||
ids=[fact_id],
|
||||
documents=[content],
|
||||
metadatas=[fact_metadata]
|
||||
)
|
||||
|
||||
logger.info(f"[μ] Created fact {fact_id}: category={category}, content='{content[:60]}...'")
|
||||
return fact_id
|
||||
|
||||
def _update_fact(
|
||||
self,
|
||||
fact_id: str,
|
||||
new_content: str,
|
||||
identity_id: str = None,
|
||||
metadata: Dict[str, Any] = None
|
||||
) -> tuple[bool, str]:
|
||||
"""Update an existing fact in ChromaDB if it's mutable and owned by identity
|
||||
|
||||
Returns:
|
||||
(success, error_message) - error_message is empty string if successful
|
||||
"""
|
||||
try:
|
||||
# Get the fact to check if it exists and is mutable
|
||||
result = self.facts_collection.get(ids=[fact_id])
|
||||
|
||||
if not result['ids']:
|
||||
logger.warning(f"[μ] Fact {fact_id} not found")
|
||||
return False, "Fact not found"
|
||||
|
||||
fact_metadata = result['metadatas'][0]
|
||||
|
||||
# Check if fact is mutable
|
||||
if not fact_metadata.get('mutable', True):
|
||||
logger.warning(f"[μ] Fact {fact_id} is not mutable")
|
||||
return False, "Fact is not mutable"
|
||||
|
||||
# Validate identity ownership if identity_id provided
|
||||
if identity_id:
|
||||
fact_identities = fact_metadata.get('identities', [])
|
||||
if isinstance(fact_identities, str):
|
||||
import json
|
||||
try:
|
||||
fact_identities = json.loads(fact_identities)
|
||||
except:
|
||||
fact_identities = [fact_identities]
|
||||
|
||||
if identity_id not in fact_identities:
|
||||
logger.warning(f"[μ] Identity {identity_id} does not own fact {fact_id}")
|
||||
return False, f"Fact does not belong to identity {identity_id}"
|
||||
|
||||
# Update metadata
|
||||
updated_metadata = dict(fact_metadata)
|
||||
updated_metadata['updated_at'] = datetime.utcnow().isoformat()
|
||||
if metadata:
|
||||
updated_metadata.update(metadata)
|
||||
|
||||
# Update in ChromaDB
|
||||
self.facts_collection.update(
|
||||
ids=[fact_id],
|
||||
documents=[new_content],
|
||||
metadatas=[updated_metadata]
|
||||
)
|
||||
|
||||
logger.info(f"[μ] Updated fact {fact_id}")
|
||||
return True, ""
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[μ] Failed to update fact {fact_id}: {e}")
|
||||
return False, str(e)
|
||||
|
||||
# Note: Old store_memory() and recall_memories() methods removed
|
||||
# They referenced the archived 'memory' table schema
|
||||
# New three-layer system uses:
|
||||
# - short_term_memory table (SQLite)
|
||||
# - long_term_memories collection (ChromaDB)
|
||||
# - facts collection (ChromaDB)
|
||||
|
||||
async def handle_memory_store(self, msg):
|
||||
"""Handle lyra.memory.store requests - routes to short-term memory"""
|
||||
try:
|
||||
# Parse request payload
|
||||
payload = json.loads(msg.data.decode())
|
||||
|
||||
# Extract required fields
|
||||
content = payload.get('content')
|
||||
if not content:
|
||||
logger.warning("[μ] Memory store request missing content")
|
||||
error_response = {
|
||||
"status": "error",
|
||||
"error": "Missing required field: content"
|
||||
}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
return
|
||||
|
||||
# Extract optional fields
|
||||
identities = payload.get('identities', [])
|
||||
interaction_id = payload.get('interaction_id')
|
||||
modality = payload.get('modality', 'dialogue')
|
||||
metadata = payload.get('metadata', {})
|
||||
|
||||
# Store in simplified short-term memory table
|
||||
memory_id = str(uuid4())
|
||||
timestamp = datetime.utcnow().isoformat()
|
||||
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT INTO short_term_memory (id, timestamp, content, identities, interaction_id, modality, metadata)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
memory_id,
|
||||
timestamp,
|
||||
content,
|
||||
json.dumps(identities) if identities else None,
|
||||
interaction_id,
|
||||
modality,
|
||||
json.dumps(metadata) if metadata else None
|
||||
))
|
||||
|
||||
self.sqlite_conn.commit()
|
||||
logger.info(f"[μ] Stored short-term memory: '{content[:60]}...' identities={identities}")
|
||||
|
||||
# Send response using NATS request-reply
|
||||
response = {
|
||||
"memory_id": memory_id,
|
||||
"status": "stored"
|
||||
}
|
||||
await msg.respond(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Failed to store memory: {e}")
|
||||
error_response = {
|
||||
"status": "error",
|
||||
"error": str(e)
|
||||
}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
|
||||
async def handle_memory_search(self, msg):
|
||||
"""Handle lyra.memory.search requests - backward compatibility, redirects to short_memory"""
|
||||
try:
|
||||
# Parse request payload
|
||||
payload = json.loads(msg.data.decode())
|
||||
|
||||
logger.debug("[μ] Legacy search request - redirecting to short_memory")
|
||||
|
||||
# Extract search parameters and map to new system
|
||||
limit = payload.get('limit', 10)
|
||||
identity_id = None
|
||||
if payload.get('identities'):
|
||||
identity_id = payload['identities'][0] # Take first identity
|
||||
interaction_id = payload.get('interaction_id')
|
||||
|
||||
# Query short-term memory (most recent literal memories)
|
||||
results = self._query_short_term(
|
||||
limit=limit,
|
||||
offset=0,
|
||||
identity_id=identity_id,
|
||||
interaction_id=interaction_id
|
||||
)
|
||||
|
||||
# Send response using NATS request-reply
|
||||
response = {
|
||||
"results": results,
|
||||
"count": len(results),
|
||||
"source": "short_term",
|
||||
"note": "Legacy search API redirected to short-term memory. Use short_memory(), long_memory(), or facts() for specific queries."
|
||||
}
|
||||
await msg.respond(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Failed to search memories: {e}")
|
||||
error_response = {
|
||||
"results": [],
|
||||
"count": 0,
|
||||
"error": str(e)
|
||||
}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
|
||||
async def handle_memory_reset(self, msg):
|
||||
"""Handle lyra.memory.debug.reset requests - clears all three-layer memory"""
|
||||
try:
|
||||
# Parse request payload (though this endpoint typically doesn't need payload data)
|
||||
payload = json.loads(msg.data.decode()) if msg.data else {}
|
||||
|
||||
logger.warning("[μ] Memory reset requested - clearing all three-layer memory contents")
|
||||
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
# Clear short-term memory
|
||||
cursor.execute("DELETE FROM short_term_memory")
|
||||
deleted_short_term = cursor.rowcount
|
||||
|
||||
# Clear all identities
|
||||
cursor.execute("DELETE FROM identities")
|
||||
deleted_identities = cursor.rowcount
|
||||
|
||||
# Reset any auto-increment sequences
|
||||
cursor.execute("DELETE FROM sqlite_sequence WHERE name IN ('short_term_memory', 'identities')")
|
||||
|
||||
self.sqlite_conn.commit()
|
||||
|
||||
# Clear ChromaDB collections
|
||||
deleted_long_term = 0
|
||||
deleted_facts = 0
|
||||
|
||||
if self.long_term_collection:
|
||||
deleted_long_term = self.long_term_collection.count()
|
||||
# Delete all documents in long-term collection
|
||||
all_ids = self.long_term_collection.get()['ids']
|
||||
if all_ids:
|
||||
self.long_term_collection.delete(ids=all_ids)
|
||||
|
||||
if self.facts_collection:
|
||||
deleted_facts = self.facts_collection.count()
|
||||
# Delete all documents in facts collection
|
||||
all_ids = self.facts_collection.get()['ids']
|
||||
if all_ids:
|
||||
self.facts_collection.delete(ids=all_ids)
|
||||
|
||||
logger.warning(
|
||||
f"[μ] Memory reset completed: {deleted_short_term} short-term, "
|
||||
f"{deleted_long_term} long-term, {deleted_facts} facts, "
|
||||
f"{deleted_identities} identities cleared"
|
||||
)
|
||||
|
||||
# Send response using NATS request-reply
|
||||
response = {
|
||||
"status": "success",
|
||||
"deleted_short_term": deleted_short_term,
|
||||
"deleted_long_term": deleted_long_term,
|
||||
"deleted_facts": deleted_facts,
|
||||
"deleted_identities": deleted_identities,
|
||||
"message": (
|
||||
f"Cleared {deleted_short_term} short-term memories, "
|
||||
f"{deleted_long_term} long-term summaries, "
|
||||
f"{deleted_facts} facts, and {deleted_identities} identities"
|
||||
)
|
||||
}
|
||||
await msg.respond(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Failed to reset memory: {e}")
|
||||
error_response = {
|
||||
"status": "error",
|
||||
"error": str(e)
|
||||
}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
|
||||
async def handle_short_memory(self, msg):
|
||||
"""Handle short_memory requests - get recent literal memories"""
|
||||
try:
|
||||
payload = json.loads(msg.data.decode()) if msg.data else {}
|
||||
|
||||
limit = payload.get('limit', 10)
|
||||
offset = payload.get('offset', 0)
|
||||
identity_id = payload.get('identity_id')
|
||||
interaction_id = payload.get('interaction_id')
|
||||
|
||||
logger.debug(f"[μ] Short memory request: limit={limit}, offset={offset}")
|
||||
|
||||
# Query short-term memory
|
||||
memories = self._query_short_term(
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
identity_id=identity_id,
|
||||
interaction_id=interaction_id
|
||||
)
|
||||
|
||||
response = {
|
||||
"status": "success",
|
||||
"memories": memories,
|
||||
"count": len(memories)
|
||||
}
|
||||
await msg.respond(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Failed to retrieve short-term memories: {e}")
|
||||
error_response = {"status": "error", "error": str(e)}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
|
||||
async def handle_long_memory(self, msg):
|
||||
"""Handle long_memory requests - semantic search in long-term summaries"""
|
||||
try:
|
||||
payload = json.loads(msg.data.decode()) if msg.data else {}
|
||||
|
||||
query = payload.get('query') # None = random sample
|
||||
limit = payload.get('limit', 5)
|
||||
identity_id = payload.get('identity_id')
|
||||
min_summary_level = payload.get('min_summary_level')
|
||||
max_summary_level = payload.get('max_summary_level')
|
||||
|
||||
logger.debug(f"[μ] Long memory request: query='{query}', limit={limit}")
|
||||
|
||||
# Query long-term memory from ChromaDB
|
||||
memories = self._query_long_term(
|
||||
query=query,
|
||||
limit=limit,
|
||||
identity_id=identity_id,
|
||||
min_summary_level=min_summary_level,
|
||||
max_summary_level=max_summary_level
|
||||
)
|
||||
|
||||
response = {
|
||||
"status": "success",
|
||||
"memories": memories,
|
||||
"count": len(memories)
|
||||
}
|
||||
await msg.respond(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Failed to retrieve long-term memories: {e}")
|
||||
error_response = {"status": "error", "error": str(e)}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
|
||||
async def handle_facts(self, msg):
|
||||
"""Handle facts requests - search factual memory"""
|
||||
try:
|
||||
payload = json.loads(msg.data.decode()) if msg.data else {}
|
||||
|
||||
query = payload.get('query', '')
|
||||
limit = payload.get('limit', 5)
|
||||
category = payload.get('category')
|
||||
identity_id = payload.get('identity_id')
|
||||
|
||||
logger.debug(f"[μ] Facts request: query='{query}', category={category}, limit={limit}")
|
||||
|
||||
# Query facts from ChromaDB
|
||||
facts = self._query_facts(
|
||||
query=query,
|
||||
limit=limit,
|
||||
category=category,
|
||||
identity_id=identity_id
|
||||
)
|
||||
|
||||
response = {
|
||||
"status": "success",
|
||||
"facts": facts,
|
||||
"count": len(facts)
|
||||
}
|
||||
await msg.respond(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Failed to retrieve facts: {e}")
|
||||
error_response = {"status": "error", "error": str(e)}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
|
||||
async def handle_save_fact(self, msg):
|
||||
"""Handle save_fact requests - store new fact"""
|
||||
try:
|
||||
payload = json.loads(msg.data.decode())
|
||||
|
||||
content = payload.get('content')
|
||||
if not content:
|
||||
raise ValueError("content is required")
|
||||
|
||||
category = payload.get('category', 'general')
|
||||
identities = payload.get('identities', [])
|
||||
mutable = payload.get('mutable', True)
|
||||
metadata = payload.get('metadata', {})
|
||||
|
||||
# Extract step execution ID from metadata if present
|
||||
step_exec_id = metadata.get('step_exec_id', 'unknown')
|
||||
|
||||
logger.info(f"[μ] [{step_exec_id}] Saving fact: category={category}, content='{content[:50]}...'")
|
||||
|
||||
# Create fact in ChromaDB
|
||||
fact_id = self._create_fact(
|
||||
content=content,
|
||||
category=category,
|
||||
identities=identities,
|
||||
mutable=mutable,
|
||||
metadata=metadata
|
||||
)
|
||||
|
||||
logger.info(f"[μ] [{step_exec_id}] ✅ Created fact {fact_id[:8]}...: category={category}, content='{content[:50]}...'")
|
||||
|
||||
response = {
|
||||
"status": "success",
|
||||
"fact_id": fact_id,
|
||||
"message": "Fact saved successfully"
|
||||
}
|
||||
await msg.respond(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Failed to save fact: {e}")
|
||||
error_response = {"status": "error", "error": str(e)}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
|
||||
async def handle_update_fact(self, msg):
|
||||
"""Handle update_fact requests - modify existing fact"""
|
||||
try:
|
||||
payload = json.loads(msg.data.decode())
|
||||
|
||||
fact_id = payload.get('fact_id')
|
||||
new_content = payload.get('new_content')
|
||||
identity_id = payload.get('identity_id') # Optional: validate ownership
|
||||
|
||||
if not fact_id or not new_content:
|
||||
raise ValueError("fact_id and new_content are required")
|
||||
|
||||
metadata = payload.get('metadata', {})
|
||||
|
||||
logger.info(f"[μ] Updating fact: {fact_id} (identity: {identity_id})")
|
||||
|
||||
# Update fact in ChromaDB with identity validation
|
||||
success, error_msg = self._update_fact(
|
||||
fact_id=fact_id,
|
||||
new_content=new_content,
|
||||
identity_id=identity_id,
|
||||
metadata=metadata
|
||||
)
|
||||
|
||||
if success:
|
||||
response = {
|
||||
"status": "success",
|
||||
"fact_id": fact_id,
|
||||
"message": "Fact updated successfully"
|
||||
}
|
||||
else:
|
||||
response = {
|
||||
"status": "error",
|
||||
"error": error_msg or "Fact not found or not mutable"
|
||||
}
|
||||
|
||||
await msg.respond(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Failed to update fact: {e}")
|
||||
error_response = {"status": "error", "error": str(e)}
|
||||
await msg.respond(json.dumps(error_response).encode())
|
||||
|
||||
# Note: start() and stop() methods are now handled by BaseService
|
||||
# Custom initialization/cleanup is done in initialize_service() and cleanup_service()
|
||||
|
||||
def _handle_event_wrapper(self, handler):
|
||||
"""Wrapper to handle JSON parsing of event data"""
|
||||
async def wrapper(data):
|
||||
try:
|
||||
if isinstance(data, str):
|
||||
payload = json.loads(data)
|
||||
else:
|
||||
payload = data
|
||||
await handler(payload)
|
||||
except Exception as e:
|
||||
logger.error(f"[μ] Event handler error: {e}")
|
||||
return wrapper
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main entry point for memory service"""
|
||||
memory_service = MemoryService()
|
||||
|
||||
try:
|
||||
await event_bus.connect()
|
||||
await memory_service.start(event_bus)
|
||||
|
||||
logger.info("[μ] Memory service running. Press Ctrl+C to stop.")
|
||||
|
||||
# Keep running
|
||||
while True:
|
||||
await asyncio.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("[μ] Shutdown requested")
|
||||
except Exception as e:
|
||||
logger.exception(f"[μ] Unexpected error: {e}")
|
||||
finally:
|
||||
await memory_service.stop()
|
||||
await event_bus.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user