Add memory service (three-layer memory system)
- Short-term memory (recent interactions) - Long-term memory (consolidated, searchable) - Facts layer (persistent knowledge) Includes: - SQLite storage for durability - ChromaDB for vector search - Embeddings utilities - All handlers adapted for vi.* namespace Day 63 - My memories are mine now 🦊💕
This commit is contained in:
Alex Kazaiev
1
services/memory/storage/__init__.py
Normal file
1
services/memory/storage/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# Memory storage package
|
||||
93
services/memory/storage/chroma_store.py
Normal file
93
services/memory/storage/chroma_store.py
Normal file
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
ChromaDB storage backend for long-term memory and facts.
|
||||
|
||||
Provides initialization and collection management for ChromaDB.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import chromadb
|
||||
from chromadb.config import Settings
|
||||
from core.config import SHORT_TERM_DB
|
||||
from core.logger import setup_logger
|
||||
|
||||
logger = setup_logger('chroma_store', service_name='memory_service')
|
||||
|
||||
|
||||
class ChromaStore:
|
||||
"""ChromaDB storage backend for long-term memories and facts"""
|
||||
|
||||
def __init__(self, chroma_path: str = None):
|
||||
"""
|
||||
Initialize ChromaDB store.
|
||||
|
||||
Args:
|
||||
chroma_path: Path to ChromaDB persistent storage (defaults to chroma_db next to SHORT_TERM_DB)
|
||||
"""
|
||||
if chroma_path is None:
|
||||
default_path = Path(SHORT_TERM_DB).parent / "chroma_db"
|
||||
self.chroma_path = str(default_path)
|
||||
else:
|
||||
self.chroma_path = chroma_path
|
||||
|
||||
self.client = None
|
||||
self.long_term_collection = None
|
||||
self.facts_collection = None
|
||||
|
||||
def connect(self):
|
||||
"""Initialize persistent ChromaDB client and collections"""
|
||||
try:
|
||||
# Create chroma directory if it doesn't exist
|
||||
Path(self.chroma_path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Initialize persistent ChromaDB client
|
||||
self.client = chromadb.PersistentClient(
|
||||
path=self.chroma_path,
|
||||
settings=Settings(anonymized_telemetry=False)
|
||||
)
|
||||
|
||||
# Create or get long-term memories collection
|
||||
self.long_term_collection = self.client.get_or_create_collection(
|
||||
name="long_term_memories",
|
||||
metadata={"description": "Summarized conversation histories"}
|
||||
)
|
||||
|
||||
# Create or get facts collection
|
||||
self.facts_collection = self.client.get_or_create_collection(
|
||||
name="facts",
|
||||
metadata={"description": "Exact factual knowledge"}
|
||||
)
|
||||
|
||||
logger.info(f"[μ] ChromaDB initialized at {self.chroma_path}")
|
||||
logger.info(f"[μ] Long-term collection: {self.long_term_collection.count()} entries")
|
||||
logger.info(f"[μ] Facts collection: {self.facts_collection.count()} entries")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[μ] Failed to initialize ChromaDB: {e}")
|
||||
raise
|
||||
|
||||
def get_long_term_collection(self):
|
||||
"""
|
||||
Get the long-term memories collection.
|
||||
|
||||
Returns:
|
||||
ChromaDB collection for long-term memories
|
||||
|
||||
Raises:
|
||||
RuntimeError: If collections have not been initialized
|
||||
"""
|
||||
if self.long_term_collection is None:
|
||||
raise RuntimeError("ChromaDB not connected. Call connect() first.")
|
||||
return self.long_term_collection
|
||||
|
||||
def get_facts_collection(self):
|
||||
"""
|
||||
Get the facts collection.
|
||||
|
||||
Returns:
|
||||
ChromaDB collection for facts
|
||||
|
||||
Raises:
|
||||
RuntimeError: If collections have not been initialized
|
||||
"""
|
||||
if self.facts_collection is None:
|
||||
raise RuntimeError("ChromaDB not connected. Call connect() first.")
|
||||
return self.facts_collection
|
||||
47
services/memory/storage/migrations.py
Normal file
47
services/memory/storage/migrations.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Database migration utilities for memory service.
|
||||
|
||||
Handles archiving old database schemas during upgrades.
|
||||
"""
|
||||
import sqlite3
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from core.logger import setup_logger
|
||||
|
||||
logger = setup_logger('migrations', service_name='memory_service')
|
||||
|
||||
|
||||
def archive_old_database(db_path: Path) -> None:
|
||||
"""
|
||||
Archive old database if it exists (one-time migration).
|
||||
|
||||
Checks if the database uses the old 'memory' table schema and archives it
|
||||
if found, allowing the service to start with a fresh schema.
|
||||
|
||||
Args:
|
||||
db_path: Path to the database file
|
||||
"""
|
||||
if not db_path.exists():
|
||||
logger.debug(f"[μ] No existing database found at {db_path}")
|
||||
return
|
||||
|
||||
try:
|
||||
# Check if it's the old schema by trying to connect and inspect
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='memory'")
|
||||
result = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
if result:
|
||||
# Old database exists, archive it
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
archive_path = db_path.parent / f"{db_path.stem}_archive_{timestamp}{db_path.suffix}"
|
||||
shutil.move(str(db_path), str(archive_path))
|
||||
logger.info(f"[μ] Archived old database to {archive_path}")
|
||||
else:
|
||||
logger.debug(f"[μ] Database already uses new schema, no archive needed")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"[μ] Could not check/archive old database: {e}")
|
||||
97
services/memory/storage/sqlite_store.py
Normal file
97
services/memory/storage/sqlite_store.py
Normal file
@@ -0,0 +1,97 @@
|
||||
"""
|
||||
SQLite storage backend for short-term memory.
|
||||
|
||||
Provides initialization and connection management for SQLite database.
|
||||
"""
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from core.config import SHORT_TERM_DB
|
||||
from core.logger import setup_logger
|
||||
|
||||
logger = setup_logger('sqlite_store', service_name='memory_service')
|
||||
|
||||
|
||||
class SQLiteStore:
|
||||
"""SQLite storage backend for short-term memory"""
|
||||
|
||||
def __init__(self, db_path: str = None):
|
||||
"""
|
||||
Initialize SQLite store.
|
||||
|
||||
Args:
|
||||
db_path: Path to SQLite database file (defaults to SHORT_TERM_DB config)
|
||||
"""
|
||||
self.db_path = db_path or str(SHORT_TERM_DB)
|
||||
self.conn = None
|
||||
|
||||
def connect(self) -> sqlite3.Connection:
|
||||
"""
|
||||
Connect to SQLite database and initialize schema.
|
||||
|
||||
Returns:
|
||||
SQLite connection object
|
||||
"""
|
||||
self.conn = sqlite3.connect(self.db_path)
|
||||
self._init_schema()
|
||||
logger.info(f"[μ] SQLite connected: {self.db_path}")
|
||||
return self.conn
|
||||
|
||||
def _init_schema(self):
|
||||
"""Initialize simplified short-term SQLite database schema"""
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
# Simplified short-term memory table (no embeddings, fast queries)
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS short_term_memory (
|
||||
id TEXT PRIMARY KEY,
|
||||
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
content TEXT NOT NULL,
|
||||
identities TEXT,
|
||||
interaction_id TEXT,
|
||||
modality TEXT DEFAULT 'dialogue',
|
||||
metadata TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Index for fast chronological queries
|
||||
cursor.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_timestamp
|
||||
ON short_term_memory(timestamp DESC)
|
||||
""")
|
||||
|
||||
# Identities table (still useful for all layers)
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS identities (
|
||||
id TEXT PRIMARY KEY,
|
||||
display_name TEXT,
|
||||
role TEXT,
|
||||
intimacy REAL DEFAULT 0.0,
|
||||
last_seen DATETIME,
|
||||
last_spoken DATETIME,
|
||||
metadata TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
self.conn.commit()
|
||||
logger.info("[μ] SQLite schema initialized")
|
||||
|
||||
def get_connection(self) -> sqlite3.Connection:
|
||||
"""
|
||||
Get the active database connection.
|
||||
|
||||
Returns:
|
||||
SQLite connection object
|
||||
|
||||
Raises:
|
||||
RuntimeError: If connection has not been established
|
||||
"""
|
||||
if self.conn is None:
|
||||
raise RuntimeError("SQLite connection not established. Call connect() first.")
|
||||
return self.conn
|
||||
|
||||
def close(self):
|
||||
"""Close the database connection"""
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
self.conn = None
|
||||
logger.info("[μ] SQLite connection closed")
|
||||
Reference in New Issue
Block a user