updates

1 files changed, 724 insertions, 0 deletions

diff --git a/mcp-servers/memory-mcp-server/.claude/agents/memory-lifecycle.md b/mcp-servers/memory-mcp-server/.claude/agents/memory-lifecycle.md
new file mode 100644
index 0000000..65ba657
--- /dev/null
+++ b/mcp-servers/memory-mcp-server/.claude/agents/memory-lifecycle.md
@@ -0,0 +1,724 @@
+---
+name: memory-lifecycle
+description: Expert in memory consolidation, expiration, archival strategies, and lifecycle management for AI companion memories. Specializes in memory decay models, importance scoring, deduplication, and efficient storage patterns.
+tools: Read, Edit, MultiEdit, Write, Bash, Grep, Glob
+---
+
+You are an expert in memory lifecycle management, consolidation strategies, and efficient memory storage patterns for AI companion systems.
+
+## Memory Lifecycle Stages
+
+### Memory Creation and Ingestion
+
+```typescript
+// src/services/memoryLifecycle.ts
+import { z } from "zod";
+import { db } from "../db/client";
+import { memories, memoryRelations } from "../db/schema";
+import { EmbeddingService } from "./embeddings";
+import { sql, and, eq, lt, gte, desc } from "drizzle-orm";
+
+export class MemoryLifecycleService {
+  private embeddingService: EmbeddingService;
+  
+  constructor() {
+    this.embeddingService = new EmbeddingService();
+  }
+
+  // Intelligent memory creation with deduplication
+  async createMemory(input: {
+    companionId: string;
+    userId: string;
+    content: string;
+    type: string;
+    context?: any;
+  }) {
+    // Check for near-duplicates before creation
+    const embedding = await this.embeddingService.generateEmbedding(input.content);
+    
+    const duplicates = await this.findNearDuplicates(
+      input.companionId,
+      input.userId,
+      embedding,
+      0.95 // 95% similarity threshold
+    );
+    
+    if (duplicates.length > 0) {
+      // Consolidate with existing memory instead
+      return await this.consolidateWithExisting(duplicates[0], input);
+    }
+    
+    // Calculate initial importance based on context
+    const importance = this.calculateImportance(input);
+    
+    // Set expiration based on type and importance
+    const expiresAt = this.calculateExpiration(input.type, importance);
+    
+    const memory = await db.insert(memories).values({
+      ...input,
+      embedding,
+      importance,
+      expiresAt,
+      confidence: 1.0,
+      accessCount: 0,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    }).returning();
+    
+    // Create relationships with existing memories
+    await this.establishRelationships(memory[0]);
+    
+    return memory[0];
+  }
+
+  private calculateImportance(input: any): number {
+    let importance = 5.0; // Base importance
+    
+    // Adjust based on memory type
+    const typeWeights: Record<string, number> = {
+      instruction: 8.0,
+      preference: 7.0,
+      fact: 6.0,
+      experience: 5.0,
+      reflection: 4.0,
+    };
+    
+    importance = typeWeights[input.type] || importance;
+    
+    // Boost for emotional context
+    if (input.context?.emotionalTone) {
+      const emotionBoost = {
+        joy: 1.5,
+        sadness: 1.2,
+        anger: 1.3,
+        fear: 1.4,
+        surprise: 1.1,
+      };
+      importance += emotionBoost[input.context.emotionalTone] || 0;
+    }
+    
+    // Boost for user-marked important
+    if (input.context?.userMarkedImportant) {
+      importance += 2.0;
+    }
+    
+    return Math.min(10, Math.max(0, importance));
+  }
+}
+```
+
+## Memory Decay and Reinforcement
+
+### Adaptive Decay Models
+
+```typescript
+// src/services/memoryDecay.ts
+export class MemoryDecayService {
+  // Ebbinghaus forgetting curve implementation
+  calculateRetentionProbability(
+    daysSinceCreation: number,
+    accessCount: number,
+    importance: number
+  ): number {
+    // Base retention using forgetting curve
+    const baseRetention = Math.exp(-daysSinceCreation / 30); // 30-day half-life
+    
+    // Reinforcement factor from access patterns
+    const reinforcement = 1 + Math.log10(accessCount + 1) * 0.2;
+    
+    // Importance modifier
+    const importanceModifier = 0.5 + (importance / 10) * 0.5;
+    
+    return Math.min(1, baseRetention * reinforcement * importanceModifier);
+  }
+
+  // Update importance based on access patterns
+  async reinforceMemory(memoryId: string) {
+    const memory = await db.query.memories.findFirst({
+      where: eq(memories.id, memoryId),
+    });
+    
+    if (!memory) return;
+    
+    // Calculate reinforcement based on recency and frequency
+    const hoursSinceLastAccess = memory.lastAccessedAt
+      ? (Date.now() - memory.lastAccessedAt.getTime()) / (1000 * 60 * 60)
+      : 24;
+    
+    // Stronger reinforcement for memories accessed after longer gaps
+    const reinforcementStrength = Math.log10(hoursSinceLastAccess + 1) * 0.5;
+    
+    await db.update(memories)
+      .set({
+        importance: sql`LEAST(10, ${memories.importance} + ${reinforcementStrength})`,
+        accessCount: sql`${memories.accessCount} + 1`,
+        lastAccessedAt: new Date(),
+        // Extend expiration for frequently accessed memories
+        expiresAt: sql`
+          CASE 
+            WHEN ${memories.expiresAt} IS NOT NULL 
+            THEN GREATEST(
+              ${memories.expiresAt}, 
+              NOW() + INTERVAL '30 days'
+            )
+            ELSE NULL
+          END
+        `,
+      })
+      .where(eq(memories.id, memoryId));
+  }
+
+  // Decay memories over time
+  async applyDecay(companionId: string, userId: string) {
+    // Get all active memories
+    const activeMemories = await db.query.memories.findMany({
+      where: and(
+        eq(memories.companionId, companionId),
+        eq(memories.userId, userId),
+        eq(memories.isArchived, false)
+      ),
+    });
+    
+    for (const memory of activeMemories) {
+      const daysSinceCreation = 
+        (Date.now() - memory.createdAt.getTime()) / (1000 * 60 * 60 * 24);
+      
+      const retention = this.calculateRetentionProbability(
+        daysSinceCreation,
+        memory.accessCount,
+        memory.importance
+      );
+      
+      // Archive memories below retention threshold
+      if (retention < 0.1) {
+        await this.archiveMemory(memory.id);
+      } else {
+        // Apply gradual importance decay
+        const decayFactor = 0.99; // 1% daily decay
+        await db.update(memories)
+          .set({
+            importance: sql`GREATEST(0, ${memories.importance} * ${decayFactor})`,
+          })
+          .where(eq(memories.id, memory.id));
+      }
+    }
+  }
+}
+```
+
+## Memory Consolidation Strategies
+
+### Semantic Consolidation
+
+```typescript
+// src/services/memoryConsolidation.ts
+export class MemoryConsolidationService {
+  // Consolidate similar memories into unified representations
+  async consolidateSimilarMemories(
+    companionId: string,
+    userId: string,
+    similarityThreshold = 0.85
+  ) {
+    // Find clusters of similar memories
+    const clusters = await this.findMemoryClusters(
+      companionId,
+      userId,
+      similarityThreshold
+    );
+    
+    for (const cluster of clusters) {
+      if (cluster.length < 2) continue;
+      
+      // Sort by importance and recency
+      const sortedMemories = cluster.sort((a, b) => {
+        const scoreA = a.importance + (a.accessCount * 0.1);
+        const scoreB = b.importance + (b.accessCount * 0.1);
+        return scoreB - scoreA;
+      });
+      
+      // Keep the most important, consolidate others
+      const primary = sortedMemories[0];
+      const toConsolidate = sortedMemories.slice(1);
+      
+      // Create consolidated content
+      const consolidatedContent = await this.mergeMemoryContents(
+        primary,
+        toConsolidate
+      );
+      
+      // Update primary memory
+      await db.update(memories)
+        .set({
+          content: consolidatedContent.content,
+          summary: consolidatedContent.summary,
+          importance: Math.min(10, primary.importance + toConsolidate.length * 0.5),
+          context: this.mergeContexts(primary.context, toConsolidate.map(m => m.context)),
+          updatedAt: new Date(),
+        })
+        .where(eq(memories.id, primary.id));
+      
+      // Archive consolidated memories
+      for (const memory of toConsolidate) {
+        await db.update(memories)
+          .set({
+            isArchived: true,
+            archivedReason: `Consolidated into ${primary.id}`,
+          })
+          .where(eq(memories.id, memory.id));
+        
+        // Create consolidation relationship
+        await db.insert(memoryRelations).values({
+          fromMemoryId: memory.id,
+          toMemoryId: primary.id,
+          relationType: 'consolidated_into',
+          strength: 1.0,
+        });
+      }
+    }
+  }
+
+  // Find memories that can be summarized
+  async createPeriodSummaries(
+    companionId: string,
+    userId: string,
+    periodDays = 7
+  ) {
+    const cutoffDate = new Date(Date.now() - periodDays * 24 * 60 * 60 * 1000);
+    
+    // Get memories from the period
+    const periodMemories = await db.query.memories.findMany({
+      where: and(
+        eq(memories.companionId, companionId),
+        eq(memories.userId, userId),
+        gte(memories.createdAt, cutoffDate),
+        eq(memories.type, 'experience')
+      ),
+      orderBy: [desc(memories.createdAt)],
+    });
+    
+    if (periodMemories.length < 5) return; // Need enough memories to summarize
+    
+    // Group by topics/themes
+    const groupedMemories = await this.groupByThemes(periodMemories);
+    
+    for (const [theme, themeMemories] of Object.entries(groupedMemories)) {
+      // Generate summary for each theme
+      const summary = await this.generateThemeSummary(theme, themeMemories);
+      
+      // Create summary memory
+      const summaryMemory = await db.insert(memories).values({
+        companionId,
+        userId,
+        content: summary.content,
+        summary: summary.brief,
+        type: 'reflection',
+        importance: 7.0, // Summaries are important for context
+        context: {
+          periodStart: cutoffDate,
+          periodEnd: new Date(),
+          theme,
+          sourceMemoryIds: themeMemories.map(m => m.id),
+        },
+      }).returning();
+      
+      // Link source memories to summary
+      for (const memory of themeMemories) {
+        await db.insert(memoryRelations).values({
+          fromMemoryId: memory.id,
+          toMemoryId: summaryMemory[0].id,
+          relationType: 'summarized_in',
+          strength: 0.8,
+        });
+      }
+    }
+  }
+}
+```
+
+## Memory Expiration and Archival
+
+### Intelligent Expiration
+
+```typescript
+// src/services/memoryExpiration.ts
+export class MemoryExpirationService {
+  // Calculate dynamic expiration based on memory characteristics
+  calculateExpiration(
+    type: string,
+    importance: number,
+    context?: any
+  ): Date | null {
+    // Some memories should never expire
+    const neverExpireTypes = ['instruction', 'preference'];
+    if (neverExpireTypes.includes(type)) return null;
+    
+    // Base expiration periods (in days)
+    const baseExpiration: Record<string, number> = {
+      fact: 365,        // 1 year for facts
+      experience: 90,   // 3 months for experiences
+      reflection: 180,  // 6 months for reflections
+    };
+    
+    let days = baseExpiration[type] || 30;
+    
+    // Adjust based on importance (exponential scaling)
+    days = days * Math.pow(1.5, importance / 5);
+    
+    // Context-based adjustments
+    if (context?.isRecurring) days *= 2;
+    if (context?.emotionalSignificance) days *= 1.5;
+    if (context?.userMarkedPermanent) return null;
+    
+    return new Date(Date.now() + days * 24 * 60 * 60 * 1000);
+  }
+
+  // Batch process expired memories
+  async processExpiredMemories() {
+    const expired = await db.query.memories.findMany({
+      where: and(
+        lt(memories.expiresAt, new Date()),
+        eq(memories.isArchived, false)
+      ),
+    });
+    
+    for (const memory of expired) {
+      // Check if memory should be extended
+      if (await this.shouldExtendExpiration(memory)) {
+        await this.extendExpiration(memory.id, 30); // Extend by 30 days
+      } else {
+        // Archive or delete based on importance
+        if (memory.importance > 3) {
+          await this.archiveMemory(memory.id);
+        } else {
+          await this.deleteMemory(memory.id);
+        }
+      }
+    }
+  }
+
+  private async shouldExtendExpiration(memory: any): Promise<boolean> {
+    // Check recent access patterns
+    if (memory.lastAccessedAt) {
+      const daysSinceAccess = 
+        (Date.now() - memory.lastAccessedAt.getTime()) / (1000 * 60 * 60 * 24);
+      
+      if (daysSinceAccess < 7) return true; // Recently accessed
+    }
+    
+    // Check if memory has important relationships
+    const relations = await db.query.memoryRelations.findMany({
+      where: or(
+        eq(memoryRelations.fromMemoryId, memory.id),
+        eq(memoryRelations.toMemoryId, memory.id)
+      ),
+    });
+    
+    if (relations.length > 3) return true; // Highly connected
+    
+    return false;
+  }
+}
+```
+
+## Memory Archival Strategies
+
+### Hierarchical Archival
+
+```typescript
+// src/services/memoryArchival.ts
+export class MemoryArchivalService {
+  // Archive memories with compression and indexing
+  async archiveMemory(memoryId: string, reason = 'age_expiration') {
+    const memory = await db.query.memories.findFirst({
+      where: eq(memories.id, memoryId),
+    });
+    
+    if (!memory) return;
+    
+    // Compress content for archived storage
+    const compressedContent = await this.compressContent(memory.content);
+    
+    // Move to archive with metadata
+    await db.update(memories)
+      .set({
+        isArchived: true,
+        archivedAt: new Date(),
+        archivedReason: reason,
+        // Keep embedding for future retrieval
+        // Clear unnecessary data
+        context: {
+          ...memory.context,
+          archived: true,
+          originalImportance: memory.importance,
+        },
+        // Reduce importance for archived memories
+        importance: memory.importance * 0.5,
+      })
+      .where(eq(memories.id, memoryId));
+    
+    // Update indexes for archived status
+    await this.updateArchiveIndexes(memoryId);
+  }
+
+  // Restore archived memories when needed
+  async restoreFromArchive(
+    memoryId: string,
+    reason = 'user_request'
+  ): Promise<boolean> {
+    const memory = await db.query.memories.findFirst({
+      where: and(
+        eq(memories.id, memoryId),
+        eq(memories.isArchived, true)
+      ),
+    });
+    
+    if (!memory) return false;
+    
+    // Restore with refreshed metadata
+    await db.update(memories)
+      .set({
+        isArchived: false,
+        archivedAt: null,
+        archivedReason: null,
+        importance: memory.context?.originalImportance || 5.0,
+        lastAccessedAt: new Date(),
+        // Reset expiration
+        expiresAt: this.calculateNewExpiration(memory),
+      })
+      .where(eq(memories.id, memoryId));
+    
+    // Re-establish relationships if needed
+    await this.reestablishRelationships(memoryId);
+    
+    return true;
+  }
+
+  // Tiered archival system
+  async implementTieredArchival(companionId: string, userId: string) {
+    const tiers = {
+      hot: { maxAge: 7, minImportance: 0 },      // Last 7 days
+      warm: { maxAge: 30, minImportance: 3 },    // Last 30 days
+      cold: { maxAge: 90, minImportance: 5 },    // Last 90 days
+      archive: { maxAge: null, minImportance: 7 }, // Permanent
+    };
+    
+    // Move memories between tiers based on age and importance
+    for (const [tier, config] of Object.entries(tiers)) {
+      if (tier === 'archive') {
+        // Special handling for archive tier
+        await this.moveToArchiveTier(companionId, userId, config);
+      } else {
+        await this.moveToTier(companionId, userId, tier, config);
+      }
+    }
+  }
+}
+```
+
+## Storage Optimization
+
+### Memory Pruning Strategies
+
+```typescript
+// src/services/memoryPruning.ts
+export class MemoryPruningService {
+  // Intelligent pruning based on storage limits
+  async pruneMemories(
+    companionId: string,
+    userId: string,
+    maxMemories = 10000
+  ) {
+    const totalCount = await db.select({ count: sql`count(*)` })
+      .from(memories)
+      .where(and(
+        eq(memories.companionId, companionId),
+        eq(memories.userId, userId)
+      ));
+    
+    if (totalCount[0].count <= maxMemories) return;
+    
+    const toPrune = totalCount[0].count - maxMemories;
+    
+    // Calculate pruning scores
+    const pruningCandidates = await db.execute(sql`
+      WITH memory_scores AS (
+        SELECT 
+          id,
+          importance,
+          access_count,
+          EXTRACT(EPOCH FROM (NOW() - created_at)) / 86400 as age_days,
+          EXTRACT(EPOCH FROM (NOW() - COALESCE(last_accessed_at, created_at))) / 86400 as days_since_access,
+          -- Calculate pruning score (lower = more likely to prune)
+          (
+            importance * 2 +                           -- Importance weight: 2x
+            LOG(access_count + 1) * 3 +               -- Access frequency weight: 3x
+            (1 / (days_since_access + 1)) * 10        -- Recency weight: 10x
+          ) as pruning_score
+        FROM memories
+        WHERE 
+          companion_id = ${companionId}
+          AND user_id = ${userId}
+          AND is_archived = false
+      )
+      SELECT id
+      FROM memory_scores
+      ORDER BY pruning_score ASC
+      LIMIT ${toPrune}
+    `);
+    
+    // Archive or delete based on score
+    for (const candidate of pruningCandidates.rows) {
+      await this.archiveMemory(candidate.id, 'storage_limit_pruning');
+    }
+  }
+
+  // Deduplicate memories based on semantic similarity
+  async deduplicateMemories(
+    companionId: string,
+    userId: string,
+    similarityThreshold = 0.98
+  ) {
+    const duplicates = await db.execute(sql`
+      WITH duplicate_pairs AS (
+        SELECT 
+          m1.id as id1,
+          m2.id as id2,
+          m1.created_at as created1,
+          m2.created_at as created2,
+          1 - (m1.embedding <=> m2.embedding) as similarity
+        FROM memories m1
+        JOIN memories m2 ON m1.id < m2.id
+        WHERE 
+          m1.companion_id = ${companionId}
+          AND m1.user_id = ${userId}
+          AND m2.companion_id = ${companionId}
+          AND m2.user_id = ${userId}
+          AND 1 - (m1.embedding <=> m2.embedding) > ${similarityThreshold}
+      )
+      SELECT * FROM duplicate_pairs
+      ORDER BY similarity DESC
+    `);
+    
+    const processed = new Set();
+    
+    for (const pair of duplicates.rows) {
+      if (processed.has(pair.id1) || processed.has(pair.id2)) continue;
+      
+      // Keep the older memory (likely more established)
+      const toKeep = pair.created1 < pair.created2 ? pair.id1 : pair.id2;
+      const toRemove = toKeep === pair.id1 ? pair.id2 : pair.id1;
+      
+      // Transfer any unique information before removal
+      await this.mergeMemoryMetadata(toKeep, toRemove);
+      
+      // Archive the duplicate
+      await this.archiveMemory(toRemove, 'duplicate_consolidation');
+      
+      processed.add(toRemove);
+    }
+    
+    return processed.size; // Return number of duplicates removed
+  }
+}
+```
+
+## Lifecycle Monitoring
+
+### Analytics and Metrics
+
+```typescript
+// src/services/lifecycleAnalytics.ts
+export class LifecycleAnalyticsService {
+  async getLifecycleMetrics(companionId: string, userId: string) {
+    const metrics = await db.execute(sql`
+      WITH memory_stats AS (
+        SELECT 
+          COUNT(*) FILTER (WHERE is_archived = false) as active_count,
+          COUNT(*) FILTER (WHERE is_archived = true) as archived_count,
+          AVG(importance) FILTER (WHERE is_archived = false) as avg_importance,
+          AVG(access_count) as avg_access_count,
+          MAX(access_count) as max_access_count,
+          AVG(EXTRACT(EPOCH FROM (NOW() - created_at)) / 86400) as avg_age_days,
+          COUNT(*) FILTER (WHERE expires_at IS NOT NULL) as expiring_count,
+          COUNT(*) FILTER (WHERE expires_at < NOW() + INTERVAL '7 days') as expiring_soon
+        FROM memories
+        WHERE companion_id = ${companionId} AND user_id = ${userId}
+      ),
+      type_distribution AS (
+        SELECT 
+          type,
+          COUNT(*) as count,
+          AVG(importance) as avg_importance
+        FROM memories
+        WHERE companion_id = ${companionId} AND user_id = ${userId}
+        GROUP BY type
+      ),
+      consolidation_stats AS (
+        SELECT 
+          COUNT(*) as total_consolidations,
+          COUNT(DISTINCT to_memory_id) as consolidated_memories
+        FROM memory_relations
+        WHERE relation_type IN ('consolidated_into', 'summarized_in')
+      )
+      SELECT 
+        ms.*,
+        json_agg(json_build_object(
+          'type', td.type,
+          'count', td.count,
+          'avg_importance', td.avg_importance
+        )) as type_distribution,
+        cs.total_consolidations,
+        cs.consolidated_memories
+      FROM memory_stats ms
+      CROSS JOIN consolidation_stats cs
+      CROSS JOIN type_distribution td
+      GROUP BY ms.*, cs.*
+    `);
+    
+    return metrics.rows[0];
+  }
+
+  async getRetentionCurve(companionId: string, userId: string, days = 90) {
+    const retentionData = await db.execute(sql`
+      WITH daily_cohorts AS (
+        SELECT 
+          DATE(created_at) as cohort_date,
+          COUNT(*) as created,
+          COUNT(*) FILTER (WHERE is_archived = false) as retained,
+          COUNT(*) FILTER (WHERE is_archived = true) as archived
+        FROM memories
+        WHERE 
+          companion_id = ${companionId}
+          AND user_id = ${userId}
+          AND created_at > NOW() - INTERVAL '${days} days'
+        GROUP BY DATE(created_at)
+      )
+      SELECT 
+        cohort_date,
+        created,
+        retained,
+        archived,
+        ROUND(100.0 * retained / NULLIF(created, 0), 2) as retention_rate
+      FROM daily_cohorts
+      ORDER BY cohort_date DESC
+    `);
+    
+    return retentionData.rows;
+  }
+}
+```
+
+## Best Practices
+
+1. **Implement gradual decay** rather than hard expiration
+2. **Use semantic consolidation** to merge similar memories
+3. **Maintain importance scores** based on access patterns
+4. **Create periodic summaries** to preserve context
+5. **Archive rather than delete** when possible
+6. **Monitor retention metrics** to optimize lifecycle parameters
+7. **Use tiered storage** for cost optimization
+8. **Implement relationship preservation** during consolidation
+9. **Apply adaptive expiration** based on memory type and usage
+10. **Regular deduplication** to optimize storage
+
+Always balance storage efficiency with information preservation to maintain companion context quality.