diff options
Diffstat (limited to 'tooling/vercel-ai-sdk/.claude/commands')
8 files changed, 2867 insertions, 0 deletions
diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-advanced-features-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-advanced-features-setup.md new file mode 100644 index 0000000..9a7a7b2 --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-advanced-features-setup.md @@ -0,0 +1,569 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Set up advanced AI SDK features including reasoning models, computer use, and cutting-edge capabilities +argument-hint: "[reasoning|computer-use|generative-ui|edge-optimization|experimental]" +--- + +## Set up Advanced AI SDK Features + +Configure cutting-edge AI SDK capabilities for: $ARGUMENTS + +### Current Advanced Features Analysis + +Existing advanced implementations: !`grep -r "experimental\|reasoning\|computer\|streamUI" . --include="*.ts" --include="*.tsx" | head -5` + +Provider-specific features: !`grep -r "o1-preview\|deepseek\|computer_20241022\|responses" . --include="*.ts" | head -5` + +Experimental configurations: !`grep -r "experimental_" . --include="*.ts" | head -5` + +### Advanced Feature Categories + +**Reasoning Models**: O1-Preview, O3-mini, DeepSeek R1 with thinking capabilities +**Computer Use**: Claude 3.5 Sonnet screen interaction and automation +**Generative UI**: Dynamic component streaming with streamUI +**Edge Optimization**: Vercel Edge Runtime performance enhancements +**Experimental**: Cutting-edge AI SDK experimental features + +### Your Task + +1. **Analyze project requirements** for advanced AI capabilities +2. **Configure reasoning models** with thinking mode and extended context +3. **Set up computer use tools** for automation and testing +4. **Implement generative UI** with dynamic component generation +5. **Optimize for edge deployment** with performance enhancements +6. **Enable experimental features** safely with proper fallbacks +7. **Add comprehensive monitoring** for advanced feature usage +8. **Create testing strategies** for cutting-edge capabilities + +### Implementation Requirements + +#### Reasoning Models Integration + +- O1-Preview and O3-mini setup with thinking tokens +- DeepSeek R1 configuration for enhanced reasoning +- Thinking mode visibility and streaming +- Extended context window management +- Reasoning-specific prompt engineering + +#### Computer Use Capabilities + +- Claude 3.5 Sonnet computer use tool setup +- Screen interaction and automation +- Browser automation and testing +- File system operations +- Cross-platform compatibility + +#### Generative UI Features + +- streamUI implementation for dynamic components +- Real-time component generation +- Interactive widget creation +- Chart and visualization streaming +- Form and dashboard generation + +### Expected Deliverables + +1. **Advanced provider configurations** with reasoning and computer use +2. **Generative UI implementation** with component streaming +3. **Edge runtime optimizations** for global deployment +4. **Experimental features setup** with safety controls +5. **Performance monitoring** for advanced capabilities +6. **Testing suite** covering all advanced features +7. **Documentation** with examples and best practices + +### Advanced Provider Setup + +#### Reasoning Models Configuration + +```typescript +// lib/reasoning-providers.ts +import { openai } from '@ai-sdk/openai'; +import { createOpenAI } from '@ai-sdk/openai'; + +// OpenAI O1 Models +export const o1Preview = openai('o1-preview', { + // Reasoning-specific configuration + experimental_reasoning: true, + experimental_thinkingMode: 'visible', + maxCompletionTokens: 32768, + temperature: 1.0, // Fixed for reasoning models +}); + +export const o3Mini = openai('o3-mini', { + experimental_reasoning: true, + experimental_thinkingTokens: true, + experimental_thinkingMode: 'visible', + maxCompletionTokens: 65536, +}); + +// DeepSeek R1 +export const deepseekR1 = createOpenAI({ + apiKey: process.env.DEEPSEEK_API_KEY, + baseURL: 'https://api.deepseek.com/v1', +})('deepseek-reasoner', { + experimental_reasoning: true, + experimental_thinkingTokens: true, + maxTokens: 8192, +}); + +// Reasoning model selector +export function selectReasoningModel(complexity: 'simple' | 'complex' | 'mathematical') { + switch (complexity) { + case 'mathematical': + return o1Preview; // Best for math and logic + case 'complex': + return o3Mini; // Good for complex reasoning + case 'simple': + return deepseekR1; // Fast for simple reasoning + default: + return o1Preview; + } +} +``` + +#### Computer Use Implementation + +```typescript +// lib/computer-use.ts +import { anthropic } from '@ai-sdk/anthropic'; +import { tool } from 'ai'; +import { z } from 'zod'; + +export const computerUseTool = anthropic.tools.computer_20241022({ + displayWidthPx: 1920, + displayHeightPx: 1080, + execute: async ({ action, coordinate, text }) => { + // Implement safe computer interactions + return await executeComputerAction(action, coordinate, text); + }, +}); + +export const browserAutomationTool = tool({ + description: 'Automate browser interactions for testing and data collection', + inputSchema: z.object({ + url: z.string().url(), + actions: z.array(z.object({ + type: z.enum(['navigate', 'click', 'type', 'wait', 'screenshot']), + selector: z.string().optional(), + text: z.string().optional(), + })), + }), + execute: async ({ url, actions }) => { + const results = []; + + for (const action of actions) { + const result = await executeBrowserAction(action, url); + results.push(result); + + if (!result.success) break; // Stop on error + } + + return { success: true, results }; + }, +}); + +// Safe computer action execution with permissions +async function executeComputerAction(action: string, coordinate?: [number, number], text?: string) { + // Security checks + const allowedActions = ['screenshot', 'click', 'type', 'scroll']; + if (!allowedActions.includes(action)) { + throw new Error(`Action not allowed: ${action}`); + } + + // Rate limiting + await checkRateLimit(`computer_${action}`); + + // Execute action based on platform + switch (action) { + case 'screenshot': + return await takeScreenshot(); + case 'click': + if (!coordinate) throw new Error('Click requires coordinates'); + return await performClick(coordinate); + case 'type': + if (!text) throw new Error('Type requires text'); + return await typeText(text); + case 'scroll': + return await performScroll(text || 'down'); + default: + throw new Error(`Unsupported action: ${action}`); + } +} +``` + +#### Generative UI Setup + +```typescript +// app/api/ui/route.ts +import { streamUI } from 'ai/rsc'; +import { anthropic } from '@ai-sdk/anthropic'; +import { z } from 'zod'; + +export async function POST(req: Request) { + const { messages } = await req.json(); + + const result = streamUI({ + model: anthropic('claude-3-sonnet-20240229'), + messages, + text: ({ content }) => <div>{content}</div>, + + tools: { + createChart: { + description: 'Generate interactive charts and visualizations', + inputSchema: z.object({ + type: z.enum(['bar', 'line', 'pie', 'scatter', 'heatmap']), + data: z.array(z.record(z.any())), + title: z.string(), + options: z.record(z.any()).optional(), + }), + generate: async ({ type, data, title, options }) => { + const { default: Chart } = await import('@/components/dynamic-chart'); + return <Chart type={type} data={data} title={title} options={options} />; + }, + }, + + createForm: { + description: 'Generate dynamic forms with validation', + inputSchema: z.object({ + fields: z.array(z.object({ + name: z.string(), + type: z.enum(['text', 'email', 'number', 'select', 'textarea']), + required: z.boolean(), + options: z.array(z.string()).optional(), + })), + title: z.string(), + onSubmit: z.string().optional(), // Callback name + }), + generate: async ({ fields, title, onSubmit }) => { + const { default: DynamicForm } = await import('@/components/dynamic-form'); + return <DynamicForm fields={fields} title={title} onSubmit={onSubmit} />; + }, + }, + + createDashboard: { + description: 'Build interactive dashboards with multiple widgets', + inputSchema: z.object({ + layout: z.enum(['grid', 'flex', 'sidebar']), + widgets: z.array(z.object({ + type: z.enum(['metric', 'chart', 'table', 'list']), + title: z.string(), + data: z.any(), + size: z.enum(['small', 'medium', 'large']).optional(), + })), + }), + generate: async ({ layout, widgets }) => { + const { default: Dashboard } = await import('@/components/dynamic-dashboard'); + return <Dashboard layout={layout} widgets={widgets} />; + }, + }, + }, + }); + + return result.toDataStreamResponse(); +} +``` + +### Edge Optimization Configuration + +```typescript +// next.config.js - Advanced edge configuration +/** @type {import('next').NextConfig} */ +const nextConfig = { + experimental: { + runtime: 'edge', + serverComponentsExternalPackages: [ + '@ai-sdk/anthropic', + '@ai-sdk/openai', + '@ai-sdk/google', + ], + // Advanced streaming + streaming: { + compression: true, + keepAlive: true, + timeout: 300000, // 5 minutes + }, + // Edge-specific features + edgeRuntime: { + unsafeEval: false, // Security + allowMiddlewareResponseBody: true, + }, + }, + + webpack: (config, { nextRuntime, isServer }) => { + if (nextRuntime === 'edge') { + // Edge runtime optimizations + config.resolve.fallback = { + ...config.resolve.fallback, + fs: false, + net: false, + tls: false, + crypto: false, + }; + + // Reduce bundle size for edge + config.externals = [ + ...(config.externals || []), + 'sharp', // Image processing + 'canvas', // Canvas operations + ]; + } + + return config; + }, + + // Advanced headers for performance + headers: async () => [ + { + source: '/api/:path*', + headers: [ + { + key: 'Cache-Control', + value: 'public, max-age=0, s-maxage=3600, stale-while-revalidate=86400', + }, + { + key: 'X-Content-Type-Options', + value: 'nosniff', + }, + { + key: 'X-Frame-Options', + value: 'DENY', + }, + { + key: 'X-XSS-Protection', + value: '1; mode=block', + }, + ], + }, + ], +}; + +module.exports = nextConfig; +``` + +### Experimental Features Configuration + +```typescript +// lib/experimental-features.ts +import { streamText, generateObject, streamUI } from 'ai'; + +export const experimentalConfig = { + // Multi-modal streaming + multimodalStreaming: true, + + // Advanced tool calling + toolCallStreaming: true, + continueSteps: true, + + // Reasoning capabilities + reasoning: true, + thinkingMode: 'visible', + thinkingTokens: true, + + // Performance optimizations + streamingTimeouts: { + streamingTimeout: 30000, + completeTimeout: 120000, + keepAliveInterval: 5000, + }, + + // Memory management + memoryManagement: { + maxTokensInMemory: 50000, + enableGarbageCollection: true, + cleanupInterval: 60000, + }, + + // Connection optimization + connectionOptimization: { + enableCompression: true, + enableKeepAlive: true, + connectionPooling: true, + }, +}; + +// Experimental feature wrapper +export function withExperimentalFeatures<T extends Function>(fn: T): T { + return (async (...args: any[]) => { + try { + // Enable experimental features for this call + const result = await fn(...args); + + // Track experimental feature usage + await trackExperimentalUsage(fn.name, true); + + return result; + } catch (error) { + // Fallback to stable version on experimental failure + console.warn(`Experimental feature ${fn.name} failed, falling back:`, error); + + await trackExperimentalUsage(fn.name, false); + + // Implement fallback logic here + throw error; // or return fallback result + } + }) as T; +} + +// Feature flag system +export class FeatureFlags { + private static flags = new Map<string, boolean>(); + + static async initialize() { + // Load feature flags from environment or external service + this.flags.set('reasoning_models', process.env.ENABLE_REASONING === 'true'); + this.flags.set('computer_use', process.env.ENABLE_COMPUTER_USE === 'true'); + this.flags.set('generative_ui', process.env.ENABLE_GENERATIVE_UI === 'true'); + this.flags.set('edge_optimization', process.env.ENABLE_EDGE_OPT === 'true'); + } + + static isEnabled(feature: string): boolean { + return this.flags.get(feature) ?? false; + } + + static enable(feature: string) { + this.flags.set(feature, true); + } + + static disable(feature: string) { + this.flags.set(feature, false); + } +} + +async function trackExperimentalUsage(feature: string, success: boolean) { + // Track experimental feature usage for monitoring + const usage = { + feature, + success, + timestamp: Date.now(), + environment: process.env.NODE_ENV, + }; + + // Send to analytics service + console.log('Experimental feature usage:', usage); +} +``` + +### Advanced Monitoring and Analytics + +```typescript +// lib/advanced-monitoring.ts +export class AdvancedMonitoring { + static async recordAdvancedMetric( + feature: string, + metric: string, + value: number, + metadata: Record<string, any> = {} + ) { + const record = { + feature, + metric, + value, + metadata, + timestamp: Date.now(), + environment: process.env.NODE_ENV, + region: process.env.VERCEL_REGION || 'unknown', + }; + + // Send to monitoring service + await this.sendToMonitoring(record); + } + + static async recordReasoningMetrics( + model: string, + thinkingTokens: number, + completionTokens: number, + success: boolean + ) { + await this.recordAdvancedMetric('reasoning', 'token_usage', thinkingTokens + completionTokens, { + model, + thinking_tokens: thinkingTokens, + completion_tokens: completionTokens, + success, + }); + } + + static async recordComputerUseMetrics( + action: string, + duration: number, + success: boolean + ) { + await this.recordAdvancedMetric('computer_use', 'action_duration', duration, { + action, + success, + }); + } + + static async recordGenerativeUIMetrics( + componentType: string, + renderTime: number, + complexity: 'low' | 'medium' | 'high' + ) { + await this.recordAdvancedMetric('generative_ui', 'render_time', renderTime, { + component_type: componentType, + complexity, + }); + } + + private static async sendToMonitoring(record: any) { + // Implementation depends on your monitoring service + // Examples: DataDog, New Relic, Custom Analytics + console.log('Advanced Monitoring:', record); + } +} +``` + +### Testing Advanced Features + +```typescript +// tests/advanced-features.test.ts +import { describe, it, expect } from 'vitest'; +import { experimentalConfig, FeatureFlags } from '@/lib/experimental-features'; + +describe('Advanced Features', () => { + beforeAll(async () => { + await FeatureFlags.initialize(); + }); + + it('should handle reasoning models', async () => { + if (!FeatureFlags.isEnabled('reasoning_models')) { + return; // Skip if not enabled + } + + const result = await testReasoningModel(); + expect(result.success).toBe(true); + expect(result.thinking_tokens).toBeGreaterThan(0); + }); + + it('should execute computer use safely', async () => { + if (!FeatureFlags.isEnabled('computer_use')) { + return; + } + + const result = await testComputerUse(); + expect(result.screenshot).toBeDefined(); + expect(result.actions).toBeInstanceOf(Array); + }); + + it('should generate UI components', async () => { + if (!FeatureFlags.isEnabled('generative_ui')) { + return; + } + + const component = await testGenerativeUI(); + expect(component).toBeDefined(); + expect(component.type).toBe('chart'); + }); +}); +``` + +### Security Considerations + +- **Feature flags**: Control advanced features with environment variables +- **Rate limiting**: Implement strict limits for resource-intensive features +- **Permissions**: Computer use requires explicit user permissions +- **Monitoring**: Track all advanced feature usage and errors +- **Fallbacks**: Always have stable alternatives for experimental features +- **Testing**: Comprehensive testing in isolated environments +- **Documentation**: Clear usage guidelines and safety measures + +Focus on building cutting-edge AI applications that push the boundaries of what's possible while maintaining security, reliability, and user safety.
\ No newline at end of file diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-chat-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-chat-setup.md new file mode 100644 index 0000000..d794c10 --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-chat-setup.md @@ -0,0 +1,58 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Set up a complete AI chat interface with streaming +argument-hint: "[basic|advanced|multimodal|rag|agent]" +--- + +## Set up AI Chat Interface + +Create a production-ready chat interface with the Vercel AI SDK based on the specified type: $ARGUMENTS + +### Project Context + +Current project structure: !`find . -type f -name "*.json" -o -name "*.ts" -o -name "*.tsx" | head -10` + +Current dependencies: !`cat package.json | jq '.dependencies // {}' 2>/dev/null || echo "No package.json found"` + +### Requirements Analysis + +Based on the requested chat type ($ARGUMENTS), I'll implement: + +**Basic Chat**: Simple text-based streaming chat interface +**Advanced Chat**: Enhanced UI with message history, error handling, and optimizations +**Multimodal Chat**: Support for images, PDFs, and file uploads +**RAG Chat**: Retrieval-augmented generation with knowledge base +**Agent Chat**: Tool-calling agents with function execution + +### Your Task + +1. **Analyze the current project structure** to understand the existing setup +2. **Install required dependencies** if not already present +3. **Create the appropriate chat implementation** based on the specified type +4. **Set up the API route** with proper streaming and error handling +5. **Implement the React component** with modern UI patterns +6. **Add proper TypeScript types** for type safety +7. **Include error boundaries** and loading states +8. **Test the implementation** and provide usage instructions + +### Implementation Guidelines + +- Use the latest AI SDK patterns and best practices +- Implement proper error handling and loading states +- Add TypeScript types for all interfaces +- Follow Next.js App Router conventions +- Include proper accessibility features +- Use modern React patterns (hooks, Suspense, etc.) +- Add responsive design considerations +- Implement proper security measures + +### Expected Deliverables + +1. API route handler (`app/api/chat/route.ts`) +2. Chat component (`components/chat.tsx` or similar) +3. Required TypeScript types +4. Updated package.json dependencies +5. Basic styling (Tailwind classes) +6. Usage documentation and examples + +Focus on creating a robust, production-ready implementation that follows AI SDK best practices and modern web development standards. diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-experimental-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-experimental-setup.md new file mode 100644 index 0000000..80d83ff --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-experimental-setup.md @@ -0,0 +1,793 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Enable cutting-edge experimental AI SDK features safely +argument-hint: "[beta|experimental|research|custom]" +--- + +## Enable Experimental AI SDK Features + +Configure and safely enable cutting-edge AI SDK experimental features: $ARGUMENTS + +### Current Experimental Status + +Existing experimental features: !`grep -r "experimental\|beta\|alpha" . --include="*.ts" --include="*.json" | head -5` + +Feature flags: !`grep -r "ENABLE_\|FLAG_" .env* 2>/dev/null | head -3 || echo "No feature flags found"` + +Advanced configurations: !`grep -r "streamingTimeouts\|thinkingMode\|toolCallStreaming" . --include="*.ts" | head -5` + +### Experimental Feature Categories + +**Beta Features**: Stable experimental features ready for production testing +**Experimental**: Cutting-edge features in active development +**Research**: Bleeding-edge research features for experimentation +**Custom**: Custom experimental implementations and modifications + +### Your Task + +1. **Analyze experimental feature landscape** and identify safe options +2. **Implement feature flag system** for controlled rollouts +3. **Configure experimental AI SDK options** with proper safeguards +4. **Set up A/B testing framework** for feature validation +5. **Add monitoring and telemetry** for experimental features +6. **Create fallback mechanisms** for experimental feature failures +7. **Implement gradual rollout strategy** with user controls +8. **Add comprehensive testing** for experimental features + +### Implementation Requirements + +#### Feature Flag System + +- Environment-based feature control +- User-level feature toggles +- Percentage-based rollouts +- Real-time feature flag updates +- Fallback mechanisms for failures + +#### Safety Measures + +- Automatic fallback to stable features +- Error isolation and reporting +- Performance impact monitoring +- User experience protection +- Data integrity guarantees + +#### Experimental Configuration + +- Advanced streaming options +- Cutting-edge model features +- Research-level AI capabilities +- Custom provider integrations +- Performance optimizations + +### Expected Deliverables + +1. **Feature flag system** with environment and user controls +2. **Experimental AI SDK configurations** with safety controls +3. **A/B testing framework** for feature validation +4. **Monitoring and telemetry** for experimental features +5. **Fallback mechanisms** for reliability +6. **Documentation** for experimental feature usage +7. **Testing suite** covering experimental scenarios + +### Feature Flag Infrastructure + +#### Core Feature Flag System + +```typescript +// lib/experimental/feature-flags.ts +interface FeatureFlag { + name: string; + enabled: boolean; + rolloutPercentage: number; + conditions?: { + userIds?: string[]; + environments?: string[]; + regions?: string[]; + custom?: (context: any) => boolean; + }; + metadata?: { + description: string; + added: string; + owner: string; + stableDate?: string; + }; +} + +export class ExperimentalFeatureManager { + private static instance: ExperimentalFeatureManager; + private flags: Map<string, FeatureFlag> = new Map(); + private context: any = {}; + + static getInstance(): ExperimentalFeatureManager { + if (!ExperimentalFeatureManager.instance) { + ExperimentalFeatureManager.instance = new ExperimentalFeatureManager(); + } + return ExperimentalFeatureManager.instance; + } + + async initialize(context: any = {}) { + this.context = context; + await this.loadFeatureFlags(); + } + + private async loadFeatureFlags() { + // Load from environment variables + const envFlags = this.loadFromEnvironment(); + + // Load from external service (optional) + const remoteFlags = await this.loadFromRemoteService(); + + // Merge flags with priority: remote > environment > defaults + const allFlags = { ...this.getDefaultFlags(), ...envFlags, ...remoteFlags }; + + Object.entries(allFlags).forEach(([name, flag]) => { + this.flags.set(name, flag as FeatureFlag); + }); + } + + private getDefaultFlags(): Record<string, FeatureFlag> { + return { + 'reasoning-models': { + name: 'reasoning-models', + enabled: false, + rolloutPercentage: 0, + metadata: { + description: 'Enable O1, O3-mini, and DeepSeek reasoning models', + added: '2024-12-01', + owner: 'ai-team', + }, + }, + 'computer-use': { + name: 'computer-use', + enabled: false, + rolloutPercentage: 0, + conditions: { + environments: ['development', 'staging'], + }, + metadata: { + description: 'Enable Claude 3.5 Sonnet computer use capabilities', + added: '2024-12-01', + owner: 'automation-team', + }, + }, + 'generative-ui': { + name: 'generative-ui', + enabled: true, + rolloutPercentage: 100, + metadata: { + description: 'Enable streamUI for dynamic component generation', + added: '2024-11-01', + owner: 'ui-team', + }, + }, + 'advanced-streaming': { + name: 'advanced-streaming', + enabled: true, + rolloutPercentage: 50, + metadata: { + description: 'Advanced streaming patterns with multi-step and waitUntil', + added: '2024-11-15', + owner: 'streaming-team', + }, + }, + 'edge-optimization': { + name: 'edge-optimization', + enabled: true, + rolloutPercentage: 75, + conditions: { + environments: ['production', 'staging'], + }, + metadata: { + description: 'Vercel Edge Runtime optimizations', + added: '2024-10-01', + owner: 'performance-team', + }, + }, + 'natural-language-sql': { + name: 'natural-language-sql', + enabled: false, + rolloutPercentage: 25, + conditions: { + custom: (context) => context.hasDatabase === true, + }, + metadata: { + description: 'Natural language to SQL conversion', + added: '2024-12-10', + owner: 'data-team', + }, + }, + }; + } + + private loadFromEnvironment(): Record<string, Partial<FeatureFlag>> { + const flags: Record<string, Partial<FeatureFlag>> = {}; + + // Load from environment variables + if (process.env.ENABLE_REASONING_MODELS === 'true') { + flags['reasoning-models'] = { enabled: true, rolloutPercentage: 100 }; + } + + if (process.env.ENABLE_COMPUTER_USE === 'true') { + flags['computer-use'] = { enabled: true, rolloutPercentage: 100 }; + } + + if (process.env.ENABLE_GENERATIVE_UI === 'true') { + flags['generative-ui'] = { enabled: true, rolloutPercentage: 100 }; + } + + if (process.env.ENABLE_ADVANCED_STREAMING === 'true') { + flags['advanced-streaming'] = { enabled: true, rolloutPercentage: 100 }; + } + + if (process.env.ENABLE_EDGE_OPTIMIZATION === 'true') { + flags['edge-optimization'] = { enabled: true, rolloutPercentage: 100 }; + } + + return flags; + } + + private async loadFromRemoteService(): Promise<Record<string, Partial<FeatureFlag>>> { + // Optional: Load from external feature flag service + try { + if (process.env.FEATURE_FLAG_SERVICE_URL) { + const response = await fetch(process.env.FEATURE_FLAG_SERVICE_URL, { + headers: { + 'Authorization': `Bearer ${process.env.FEATURE_FLAG_API_KEY}`, + }, + }); + + if (response.ok) { + return await response.json(); + } + } + } catch (error) { + console.warn('Failed to load remote feature flags:', error); + } + + return {}; + } + + isEnabled(flagName: string, userId?: string): boolean { + const flag = this.flags.get(flagName); + if (!flag) return false; + + // Check basic enabled status + if (!flag.enabled) return false; + + // Check conditions + if (flag.conditions) { + if (flag.conditions.userIds && userId) { + if (!flag.conditions.userIds.includes(userId)) return false; + } + + if (flag.conditions.environments) { + const env = process.env.NODE_ENV || 'development'; + if (!flag.conditions.environments.includes(env)) return false; + } + + if (flag.conditions.regions) { + const region = process.env.VERCEL_REGION || 'local'; + if (!flag.conditions.regions.includes(region)) return false; + } + + if (flag.conditions.custom) { + if (!flag.conditions.custom(this.context)) return false; + } + } + + // Check rollout percentage + if (flag.rolloutPercentage < 100) { + const hash = this.getUserHash(userId || 'anonymous', flagName); + if (hash % 100 >= flag.rolloutPercentage) return false; + } + + return true; + } + + private getUserHash(userId: string, flagName: string): number { + // Simple hash function for consistent user bucketing + let hash = 0; + const str = `${userId}-${flagName}`; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32-bit integer + } + return Math.abs(hash); + } + + getAllFlags(): Map<string, FeatureFlag> { + return new Map(this.flags); + } + + updateFlag(flagName: string, updates: Partial<FeatureFlag>) { + const existing = this.flags.get(flagName); + if (existing) { + this.flags.set(flagName, { ...existing, ...updates }); + } + } + + async trackFeatureUsage(flagName: string, userId?: string, metadata?: any) { + const usage = { + flag: flagName, + userId, + timestamp: Date.now(), + context: this.context, + metadata, + }; + + // Send to analytics service + await this.sendUsageToAnalytics(usage); + } + + private async sendUsageToAnalytics(usage: any) { + try { + if (process.env.ANALYTICS_ENDPOINT) { + await fetch(process.env.ANALYTICS_ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(usage), + }); + } + } catch (error) { + console.warn('Failed to send feature usage analytics:', error); + } + } +} + +// Singleton instance +export const featureFlags = ExperimentalFeatureManager.getInstance(); +``` + +#### Experimental AI SDK Wrapper + +```typescript +// lib/experimental/ai-sdk-experimental.ts +import { streamText, generateText, streamUI, generateObject } from 'ai'; +import { featureFlags } from './feature-flags'; + +export interface ExperimentalOptions { + userId?: string; + fallbackOnError?: boolean; + trackUsage?: boolean; +} + +export class ExperimentalAISDK { + + static async streamText(config: any, options: ExperimentalOptions = {}) { + const { userId, fallbackOnError = true, trackUsage = true } = options; + + // Apply experimental features based on flags + const experimentalConfig = await this.applyExperimentalFeatures(config, userId); + + try { + const result = streamText(experimentalConfig); + + if (trackUsage) { + await this.trackExperimentalUsage(experimentalConfig, userId); + } + + return result; + } catch (error) { + if (fallbackOnError) { + console.warn('Experimental feature failed, falling back to stable:', error); + return streamText(config); // Fallback to original config + } + throw error; + } + } + + static async generateText(config: any, options: ExperimentalOptions = {}) { + const { userId, fallbackOnError = true, trackUsage = true } = options; + + const experimentalConfig = await this.applyExperimentalFeatures(config, userId); + + try { + const result = await generateText(experimentalConfig); + + if (trackUsage) { + await this.trackExperimentalUsage(experimentalConfig, userId); + } + + return result; + } catch (error) { + if (fallbackOnError) { + console.warn('Experimental feature failed, falling back to stable:', error); + return generateText(config); + } + throw error; + } + } + + static async streamUI(config: any, options: ExperimentalOptions = {}) { + const { userId, fallbackOnError = true, trackUsage = true } = options; + + if (!featureFlags.isEnabled('generative-ui', userId)) { + throw new Error('Generative UI is not enabled for this user'); + } + + try { + const result = streamUI(config); + + if (trackUsage) { + await featureFlags.trackFeatureUsage('generative-ui', userId, { + toolCount: Object.keys(config.tools || {}).length, + }); + } + + return result; + } catch (error) { + if (fallbackOnError) { + // Fallback to regular text streaming + console.warn('StreamUI failed, falling back to streamText:', error); + return streamText({ + ...config, + text: ({ content }) => content, // Simple text output + }); + } + throw error; + } + } + + private static async applyExperimentalFeatures(config: any, userId?: string) { + const experimentalConfig = { ...config }; + + // Advanced streaming features + if (featureFlags.isEnabled('advanced-streaming', userId)) { + experimentalConfig.experimental_streamingTimeouts = { + streamingTimeout: 45000, + completeTimeout: 120000, + keepAliveInterval: 5000, + }; + + experimentalConfig.experimental_toolCallStreaming = true; + experimentalConfig.experimental_continueSteps = true; + + await featureFlags.trackFeatureUsage('advanced-streaming', userId); + } + + // Reasoning models + if (featureFlags.isEnabled('reasoning-models', userId)) { + if (config.model?.includes?.('o1') || config.model?.includes?.('reasoner')) { + experimentalConfig.experimental_reasoning = true; + experimentalConfig.experimental_thinkingMode = 'visible'; + experimentalConfig.experimental_thinkingTokens = true; + + await featureFlags.trackFeatureUsage('reasoning-models', userId); + } + } + + // Edge optimizations + if (featureFlags.isEnabled('edge-optimization', userId)) { + experimentalConfig.experimental_edgeOptimization = { + enableCompression: true, + enableKeepAlive: true, + connectionPooling: true, + }; + + experimentalConfig.experimental_memoryManagement = { + maxTokensInMemory: 25000, + enableGarbageCollection: true, + cleanupInterval: 30000, + }; + + await featureFlags.trackFeatureUsage('edge-optimization', userId); + } + + return experimentalConfig; + } + + private static async trackExperimentalUsage(config: any, userId?: string) { + const experimentalFeatures = []; + + if (config.experimental_streamingTimeouts) { + experimentalFeatures.push('advanced-streaming'); + } + + if (config.experimental_reasoning) { + experimentalFeatures.push('reasoning-models'); + } + + if (config.experimental_edgeOptimization) { + experimentalFeatures.push('edge-optimization'); + } + + for (const feature of experimentalFeatures) { + await featureFlags.trackFeatureUsage(feature, userId, { + configuration: Object.keys(config).filter(k => k.startsWith('experimental_')), + }); + } + } +} +``` + +### A/B Testing Framework + +#### Experiment Configuration + +```typescript +// lib/experimental/ab-testing.ts +export interface Experiment { + id: string; + name: string; + description: string; + status: 'draft' | 'running' | 'paused' | 'completed'; + variants: { + id: string; + name: string; + percentage: number; + config: any; + }[]; + targetAudience?: { + userIds?: string[]; + percentage?: number; + conditions?: any; + }; + metrics: string[]; + startDate: Date; + endDate?: Date; +} + +export class ExperimentManager { + private static instance: ExperimentManager; + private experiments: Map<string, Experiment> = new Map(); + + static getInstance(): ExperimentManager { + if (!ExperimentManager.instance) { + ExperimentManager.instance = new ExperimentManager(); + } + return ExperimentManager.instance; + } + + async initialize() { + await this.loadExperiments(); + } + + private async loadExperiments() { + // Load experiments from configuration + const defaultExperiments: Experiment[] = [ + { + id: 'reasoning-vs-standard', + name: 'Reasoning Models vs Standard Models', + description: 'Compare performance of O1 reasoning models vs standard models', + status: 'running', + variants: [ + { id: 'control', name: 'Standard Model', percentage: 50, config: { useReasoning: false } }, + { id: 'treatment', name: 'Reasoning Model', percentage: 50, config: { useReasoning: true } }, + ], + targetAudience: { percentage: 10 }, + metrics: ['response_quality', 'latency', 'cost', 'user_satisfaction'], + startDate: new Date('2024-12-01'), + endDate: new Date('2024-12-31'), + }, + { + id: 'streaming-optimization', + name: 'Advanced Streaming vs Basic Streaming', + description: 'Test advanced streaming features vs basic streaming', + status: 'running', + variants: [ + { id: 'control', name: 'Basic Streaming', percentage: 70, config: { advancedStreaming: false } }, + { id: 'treatment', name: 'Advanced Streaming', percentage: 30, config: { advancedStreaming: true } }, + ], + metrics: ['latency', 'error_rate', 'user_engagement'], + startDate: new Date('2024-11-15'), + endDate: new Date('2024-12-15'), + }, + ]; + + defaultExperiments.forEach(exp => { + this.experiments.set(exp.id, exp); + }); + } + + getVariant(experimentId: string, userId: string): any { + const experiment = this.experiments.get(experimentId); + if (!experiment || experiment.status !== 'running') { + return null; + } + + // Check if user is in target audience + if (!this.isUserInAudience(experiment, userId)) { + return null; + } + + // Determine variant based on user hash + const hash = this.getUserHash(userId, experimentId); + let cumulativePercentage = 0; + + for (const variant of experiment.variants) { + cumulativePercentage += variant.percentage; + if (hash % 100 < cumulativePercentage) { + return variant; + } + } + + return experiment.variants[0]; // Fallback to first variant + } + + private isUserInAudience(experiment: Experiment, userId: string): boolean { + if (!experiment.targetAudience) return true; + + if (experiment.targetAudience.userIds) { + return experiment.targetAudience.userIds.includes(userId); + } + + if (experiment.targetAudience.percentage) { + const hash = this.getUserHash(userId, experiment.id); + return (hash % 100) < experiment.targetAudience.percentage; + } + + return true; + } + + private getUserHash(userId: string, experimentId: string): number { + let hash = 0; + const str = `${userId}-${experimentId}`; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; + } + return Math.abs(hash); + } + + async recordMetric(experimentId: string, userId: string, metric: string, value: number) { + const variant = this.getVariant(experimentId, userId); + if (!variant) return; + + const record = { + experimentId, + variantId: variant.id, + userId, + metric, + value, + timestamp: Date.now(), + }; + + await this.sendMetricToAnalytics(record); + } + + private async sendMetricToAnalytics(record: any) { + try { + if (process.env.EXPERIMENT_ANALYTICS_ENDPOINT) { + await fetch(process.env.EXPERIMENT_ANALYTICS_ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(record), + }); + } + } catch (error) { + console.warn('Failed to send experiment metric:', error); + } + } +} + +export const experiments = ExperimentManager.getInstance(); +``` + +### API Integration + +#### Experimental API Route + +```typescript +// app/api/experimental/chat/route.ts +import { ExperimentalAISDK } from '@/lib/experimental/ai-sdk-experimental'; +import { experiments } from '@/lib/experimental/ab-testing'; +import { anthropic } from '@ai-sdk/anthropic'; + +export const runtime = 'edge'; +export const maxDuration = 300; + +export async function POST(req: Request) { + const { messages, userId } = await req.json(); + + try { + // Get experiment variant + const reasoningExperiment = experiments.getVariant('reasoning-vs-standard', userId); + const streamingExperiment = experiments.getVariant('streaming-optimization', userId); + + // Configure based on experiments + const config = { + model: reasoningExperiment?.config.useReasoning + ? anthropic('claude-3-sonnet-20240229') // Would use O1 in real implementation + : anthropic('claude-3-sonnet-20240229'), + messages, + }; + + // Use experimental SDK + const result = await ExperimentalAISDK.streamText(config, { + userId, + fallbackOnError: true, + trackUsage: true, + }); + + // Record experiment metrics + if (reasoningExperiment) { + // This would be implemented with actual metrics + await experiments.recordMetric('reasoning-vs-standard', userId, 'request_count', 1); + } + + return result.toUIMessageStreamResponse(); + + } catch (error) { + console.error('Experimental chat error:', error); + + // Fallback to stable implementation + const result = await ExperimentalAISDK.streamText({ + model: anthropic('claude-3-sonnet-20240229'), + messages, + }, { userId, fallbackOnError: false }); + + return result.toUIMessageStreamResponse(); + } +} +``` + +### Monitoring and Safety + +#### Experimental Feature Monitoring + +```typescript +// lib/experimental/monitoring.ts +export class ExperimentalMonitoring { + static async recordFeaturePerformance( + featureName: string, + metrics: { + latency?: number; + errorRate?: number; + userSatisfaction?: number; + cost?: number; + }, + userId?: string + ) { + const record = { + feature: featureName, + metrics, + userId, + timestamp: Date.now(), + environment: process.env.NODE_ENV, + version: process.env.APP_VERSION || 'unknown', + }; + + await this.sendToMonitoring(record); + } + + static async checkFeatureHealth(featureName: string): Promise<boolean> { + // Check if feature is performing within acceptable parameters + const healthData = await this.getFeatureHealthData(featureName); + + return ( + healthData.errorRate < 0.05 && // Less than 5% error rate + healthData.latency < 5000 && // Less than 5 second latency + healthData.userSatisfaction > 0.7 // Above 70% satisfaction + ); + } + + private static async sendToMonitoring(record: any) { + // Send to monitoring service + console.log('Experimental Feature Metrics:', record); + } + + private static async getFeatureHealthData(featureName: string) { + // This would fetch real health data from monitoring service + return { + errorRate: Math.random() * 0.1, + latency: Math.random() * 3000, + userSatisfaction: 0.7 + (Math.random() * 0.3), + }; + } +} +``` + +### Usage and Safety Guidelines + +- **Feature flags**: Control experimental features with environment variables and remote configuration +- **A/B testing**: Run controlled experiments to validate new features +- **Fallback mechanisms**: Always provide stable alternatives for experimental features +- **Monitoring**: Track performance and user experience of experimental features +- **Gradual rollout**: Start with small percentages and increase based on performance +- **Safety checks**: Implement automatic feature disabling for poor performance +- **Documentation**: Document experimental features and their expected behavior + +Focus on building a robust experimental framework that enables safe innovation while protecting user experience and system stability.
\ No newline at end of file diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-monitoring-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-monitoring-setup.md new file mode 100644 index 0000000..ee3baac --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-monitoring-setup.md @@ -0,0 +1,807 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Set up comprehensive monitoring and analytics for AI SDK applications +argument-hint: "[performance|usage|costs|errors|analytics|dashboard]" +--- + +## Set up AI SDK Monitoring and Analytics + +Configure comprehensive monitoring and analytics for AI SDK applications: $ARGUMENTS + +### Current Monitoring Analysis + +Existing monitoring setup: !`grep -r "monitoring\|analytics\|metrics" . --include="*.ts" --include="*.tsx" | head -5` + +Performance tracking: !`grep -r "performance\|latency\|duration" . --include="*.ts" | head -5` + +Error handling: !`grep -r "error\|catch\|throw" . --include="*.ts" | head -5` + +### Monitoring Categories + +**Performance**: Latency, throughput, response times, edge performance +**Usage**: Token consumption, request patterns, user behavior +**Costs**: Provider costs, usage optimization, budget tracking +**Errors**: Error rates, failure patterns, recovery metrics +**Analytics**: User insights, feature adoption, performance trends +**Dashboard**: Real-time monitoring, alerts, visualization + +### Your Task + +1. **Analyze monitoring requirements** for comprehensive AI SDK observability +2. **Implement performance tracking** with latency and throughput metrics +3. **Set up usage analytics** for token consumption and cost tracking +4. **Configure error monitoring** with detailed error classification +5. **Build real-time dashboard** for monitoring AI SDK applications +6. **Add alerting system** for performance and error thresholds +7. **Create analytics reports** for insights and optimization +8. **Integrate with external services** (DataDog, New Relic, etc.) + +### Implementation Requirements + +#### Performance Monitoring + +- Request/response latency tracking +- Streaming performance metrics +- Provider response time monitoring +- Edge runtime performance tracking +- Memory and CPU usage monitoring + +#### Usage Analytics + +- Token consumption tracking by provider and model +- Request volume and patterns +- User behavior and feature adoption +- Geographic usage distribution +- Time-based usage patterns + +#### Cost Management + +- Real-time cost calculation across providers +- Budget tracking and alerting +- Cost optimization recommendations +- Provider cost comparison +- Usage forecasting and planning + +### Expected Deliverables + +1. **Performance monitoring system** with real-time metrics +2. **Usage analytics dashboard** with cost tracking +3. **Error monitoring and alerting** with detailed classification +4. **Custom analytics implementation** for AI SDK specific metrics +5. **Integration setup** for external monitoring services +6. **Real-time dashboard** with visualizations and alerts +7. **Monitoring documentation** with setup and usage guides + +### Performance Monitoring Implementation + +#### Core Monitoring Infrastructure + +```typescript +// lib/monitoring/core.ts +import { performance } from 'perf_hooks'; + +export interface MetricData { + name: string; + value: number; + timestamp: number; + tags: Record<string, string>; + metadata?: Record<string, any>; +} + +export interface PerformanceMetrics { + latency: number; + tokens: { + input: number; + output: number; + total: number; + }; + cost: number; + provider: string; + model: string; + success: boolean; + errorType?: string; +} + +export class AISDKMonitor { + private static instance: AISDKMonitor; + private metrics: MetricData[] = []; + private performanceData: Map<string, PerformanceMetrics> = new Map(); + + static getInstance(): AISDKMonitor { + if (!AISDKMonitor.instance) { + AISDKMonitor.instance = new AISDKMonitor(); + } + return AISDKMonitor.instance; + } + + // Record basic metrics + recordMetric(name: string, value: number, tags: Record<string, string> = {}) { + const metric: MetricData = { + name, + value, + timestamp: Date.now(), + tags: { + ...tags, + environment: process.env.NODE_ENV || 'development', + region: process.env.VERCEL_REGION || 'local', + }, + }; + + this.metrics.push(metric); + this.sendToExternalServices(metric); + } + + // Record comprehensive performance metrics + recordPerformance(requestId: string, metrics: PerformanceMetrics) { + this.performanceData.set(requestId, metrics); + + // Record individual metrics + this.recordMetric('ai_request_latency', metrics.latency, { + provider: metrics.provider, + model: metrics.model, + success: metrics.success.toString(), + }); + + this.recordMetric('ai_token_usage', metrics.tokens.total, { + provider: metrics.provider, + model: metrics.model, + type: 'total', + }); + + this.recordMetric('ai_request_cost', metrics.cost, { + provider: metrics.provider, + model: metrics.model, + }); + + if (!metrics.success && metrics.errorType) { + this.recordMetric('ai_error_count', 1, { + provider: metrics.provider, + model: metrics.model, + error_type: metrics.errorType, + }); + } + } + + // Get performance analytics + getPerformanceAnalytics(timeRange: { start: Date; end: Date }) { + const filteredMetrics = this.metrics.filter(m => + m.timestamp >= timeRange.start.getTime() && + m.timestamp <= timeRange.end.getTime() + ); + + return { + totalRequests: filteredMetrics.filter(m => m.name === 'ai_request_latency').length, + averageLatency: this.calculateAverage(filteredMetrics, 'ai_request_latency'), + totalTokens: this.calculateSum(filteredMetrics, 'ai_token_usage'), + totalCost: this.calculateSum(filteredMetrics, 'ai_request_cost'), + errorRate: this.calculateErrorRate(filteredMetrics), + providerBreakdown: this.getProviderBreakdown(filteredMetrics), + }; + } + + private calculateAverage(metrics: MetricData[], metricName: string): number { + const relevant = metrics.filter(m => m.name === metricName); + if (relevant.length === 0) return 0; + return relevant.reduce((sum, m) => sum + m.value, 0) / relevant.length; + } + + private calculateSum(metrics: MetricData[], metricName: string): number { + return metrics + .filter(m => m.name === metricName) + .reduce((sum, m) => sum + m.value, 0); + } + + private calculateErrorRate(metrics: MetricData[]): number { + const totalRequests = metrics.filter(m => m.name === 'ai_request_latency').length; + const errors = metrics.filter(m => m.name === 'ai_error_count').length; + return totalRequests > 0 ? errors / totalRequests : 0; + } + + private getProviderBreakdown(metrics: MetricData[]) { + const providers = new Map<string, { requests: number; tokens: number; cost: number }>(); + + metrics.forEach(metric => { + const provider = metric.tags.provider; + if (!provider) return; + + if (!providers.has(provider)) { + providers.set(provider, { requests: 0, tokens: 0, cost: 0 }); + } + + const data = providers.get(provider)!; + + switch (metric.name) { + case 'ai_request_latency': + data.requests += 1; + break; + case 'ai_token_usage': + data.tokens += metric.value; + break; + case 'ai_request_cost': + data.cost += metric.value; + break; + } + }); + + return Object.fromEntries(providers); + } + + private sendToExternalServices(metric: MetricData) { + // Send to various monitoring services + if (process.env.DATADOG_API_KEY) { + this.sendToDataDog(metric); + } + + if (process.env.NEW_RELIC_LICENSE_KEY) { + this.sendToNewRelic(metric); + } + + if (process.env.CUSTOM_ANALYTICS_ENDPOINT) { + this.sendToCustomAnalytics(metric); + } + } + + private async sendToDataDog(metric: MetricData) { + // DataDog implementation + try { + const response = await fetch('https://api.datadoghq.com/api/v1/series', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'DD-API-KEY': process.env.DATADOG_API_KEY!, + }, + body: JSON.stringify({ + series: [{ + metric: `aisdk.${metric.name}`, + points: [[metric.timestamp / 1000, metric.value]], + tags: Object.entries(metric.tags).map(([k, v]) => `${k}:${v}`), + }], + }), + }); + + if (!response.ok) { + console.error('Failed to send metric to DataDog:', response.statusText); + } + } catch (error) { + console.error('DataDog metric send error:', error); + } + } + + private async sendToNewRelic(metric: MetricData) { + // New Relic implementation + try { + const response = await fetch('https://metric-api.newrelic.com/metric/v1', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Api-Key': process.env.NEW_RELIC_LICENSE_KEY!, + }, + body: JSON.stringify([{ + metrics: [{ + name: `aisdk.${metric.name}`, + type: 'gauge', + value: metric.value, + timestamp: metric.timestamp, + attributes: metric.tags, + }], + }]), + }); + + if (!response.ok) { + console.error('Failed to send metric to New Relic:', response.statusText); + } + } catch (error) { + console.error('New Relic metric send error:', error); + } + } + + private async sendToCustomAnalytics(metric: MetricData) { + // Custom analytics endpoint + try { + await fetch(process.env.CUSTOM_ANALYTICS_ENDPOINT!, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(metric), + }); + } catch (error) { + console.error('Custom analytics send error:', error); + } + } +} + +// Singleton instance +export const monitor = AISDKMonitor.getInstance(); +``` + +#### AI SDK Integration Middleware + +```typescript +// lib/monitoring/middleware.ts +import { streamText, generateText } from 'ai'; +import { monitor, PerformanceMetrics } from './core'; +import { calculateCost } from './cost-calculator'; + +export function withMonitoring<T extends Function>(fn: T, context: string): T { + return (async (...args: any[]) => { + const requestId = generateRequestId(); + const startTime = performance.now(); + + try { + const result = await fn(...args); + + // Extract metrics from result + const endTime = performance.now(); + const latency = endTime - startTime; + + const metrics: PerformanceMetrics = { + latency, + tokens: extractTokenUsage(result), + cost: calculateCost(extractTokenUsage(result), extractModelInfo(result)), + provider: extractProvider(result) || 'unknown', + model: extractModel(result) || 'unknown', + success: true, + }; + + monitor.recordPerformance(requestId, metrics); + + return result; + + } catch (error) { + const endTime = performance.now(); + const latency = endTime - startTime; + + const metrics: PerformanceMetrics = { + latency, + tokens: { input: 0, output: 0, total: 0 }, + cost: 0, + provider: 'unknown', + model: 'unknown', + success: false, + errorType: error.constructor.name, + }; + + monitor.recordPerformance(requestId, metrics); + + throw error; + } + }) as T; +} + +// Enhanced streaming with monitoring +export const monitoredStreamText = withMonitoring(streamText, 'stream_text'); +export const monitoredGenerateText = withMonitoring(generateText, 'generate_text'); + +function generateRequestId(): string { + return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; +} + +function extractTokenUsage(result: any) { + if (result?.usage) { + return { + input: result.usage.promptTokens || 0, + output: result.usage.completionTokens || 0, + total: result.usage.totalTokens || 0, + }; + } + return { input: 0, output: 0, total: 0 }; +} + +function extractProvider(result: any): string | null { + // Extract provider from model configuration + if (result?.model?._provider) { + return result.model._provider; + } + return null; +} + +function extractModel(result: any): string | null { + if (result?.model?.modelId) { + return result.model.modelId; + } + return null; +} + +function extractModelInfo(result: any) { + return { + provider: extractProvider(result), + model: extractModel(result), + }; +} +``` + +#### Cost Calculation System + +```typescript +// lib/monitoring/cost-calculator.ts +interface ProviderPricing { + [model: string]: { + input: number; // Cost per 1K input tokens + output: number; // Cost per 1K output tokens + }; +} + +const PROVIDER_PRICING: Record<string, ProviderPricing> = { + anthropic: { + 'claude-3-haiku-20240307': { input: 0.00025, output: 0.00125 }, + 'claude-3-sonnet-20240229': { input: 0.003, output: 0.015 }, + 'claude-3-opus-20240229': { input: 0.015, output: 0.075 }, + 'claude-3-5-sonnet-20241022': { input: 0.003, output: 0.015 }, + }, + openai: { + 'gpt-3.5-turbo': { input: 0.0015, output: 0.002 }, + 'gpt-4': { input: 0.03, output: 0.06 }, + 'gpt-4-turbo': { input: 0.01, output: 0.03 }, + 'gpt-4o': { input: 0.005, output: 0.015 }, + 'o1-preview': { input: 0.015, output: 0.06 }, + 'o1-mini': { input: 0.003, output: 0.012 }, + }, + google: { + 'gemini-pro': { input: 0.0005, output: 0.0015 }, + 'gemini-pro-vision': { input: 0.0005, output: 0.0015 }, + }, + cohere: { + 'command': { input: 0.0015, output: 0.002 }, + 'command-light': { input: 0.0003, output: 0.0006 }, + }, +}; + +export function calculateCost( + tokens: { input: number; output: number; total: number }, + modelInfo: { provider: string | null; model: string | null } +): number { + if (!modelInfo.provider || !modelInfo.model) { + return 0; + } + + const pricing = PROVIDER_PRICING[modelInfo.provider]?.[modelInfo.model]; + if (!pricing) { + console.warn(`No pricing data for ${modelInfo.provider}:${modelInfo.model}`); + return 0; + } + + const inputCost = (tokens.input / 1000) * pricing.input; + const outputCost = (tokens.output / 1000) * pricing.output; + + return inputCost + outputCost; +} + +export class CostTracker { + private static dailyCosts = new Map<string, number>(); + private static monthlyCosts = new Map<string, number>(); + + static recordCost(cost: number, provider: string, model: string) { + const today = new Date().toISOString().split('T')[0]; + const month = today.substring(0, 7); + + // Daily tracking + const dailyKey = `${today}:${provider}:${model}`; + this.dailyCosts.set(dailyKey, (this.dailyCosts.get(dailyKey) || 0) + cost); + + // Monthly tracking + const monthlyKey = `${month}:${provider}:${model}`; + this.monthlyCosts.set(monthlyKey, (this.monthlyCosts.get(monthlyKey) || 0) + cost); + + // Check budget alerts + this.checkBudgetAlerts(cost, provider); + } + + static getDailyCost(date?: string): number { + const targetDate = date || new Date().toISOString().split('T')[0]; + let total = 0; + + for (const [key, cost] of this.dailyCosts.entries()) { + if (key.startsWith(targetDate)) { + total += cost; + } + } + + return total; + } + + static getMonthlyCost(month?: string): number { + const targetMonth = month || new Date().toISOString().substring(0, 7); + let total = 0; + + for (const [key, cost] of this.monthlyCosts.entries()) { + if (key.startsWith(targetMonth)) { + total += cost; + } + } + + return total; + } + + static getProviderBreakdown(timeRange: 'daily' | 'monthly' = 'daily') { + const costs = timeRange === 'daily' ? this.dailyCosts : this.monthlyCosts; + const breakdown = new Map<string, number>(); + + for (const [key, cost] of costs.entries()) { + const provider = key.split(':')[1]; + breakdown.set(provider, (breakdown.get(provider) || 0) + cost); + } + + return Object.fromEntries(breakdown); + } + + private static checkBudgetAlerts(cost: number, provider: string) { + const dailyBudget = parseFloat(process.env.DAILY_AI_BUDGET || '50'); + const monthlyBudget = parseFloat(process.env.MONTHLY_AI_BUDGET || '1000'); + + const dailyCost = this.getDailyCost(); + const monthlyCost = this.getMonthlyCost(); + + if (dailyCost > dailyBudget * 0.9) { + this.sendBudgetAlert('daily', dailyCost, dailyBudget); + } + + if (monthlyCost > monthlyBudget * 0.9) { + this.sendBudgetAlert('monthly', monthlyCost, monthlyBudget); + } + } + + private static sendBudgetAlert(period: string, current: number, budget: number) { + const alert = { + type: 'budget_alert', + period, + current_cost: current, + budget, + utilization: current / budget, + timestamp: new Date().toISOString(), + }; + + // Send alert (email, Slack, etc.) + console.warn('BUDGET ALERT:', alert); + + // You could integrate with notification services here + if (process.env.SLACK_WEBHOOK_URL) { + this.sendSlackAlert(alert); + } + } + + private static async sendSlackAlert(alert: any) { + try { + await fetch(process.env.SLACK_WEBHOOK_URL!, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: `🚨 AI Budget Alert: ${alert.period} spending (${alert.current_cost.toFixed(2)}) is at ${(alert.utilization * 100).toFixed(1)}% of budget ($${alert.budget})`, + }), + }); + } catch (error) { + console.error('Failed to send Slack alert:', error); + } + } +} +``` + +### Real-Time Dashboard Implementation + +```typescript +// app/api/monitoring/dashboard/route.ts +export async function GET(req: Request) { + const monitor = AISDKMonitor.getInstance(); + const { searchParams } = new URL(req.url); + const timeRange = searchParams.get('range') || '1h'; + + const endTime = new Date(); + const startTime = new Date(endTime.getTime() - parseTimeRange(timeRange)); + + const analytics = monitor.getPerformanceAnalytics({ start: startTime, end: endTime }); + const costBreakdown = CostTracker.getProviderBreakdown('daily'); + + const dashboard = { + timeRange, + overview: { + totalRequests: analytics.totalRequests, + averageLatency: Math.round(analytics.averageLatency), + totalTokens: analytics.totalTokens, + totalCost: analytics.totalCost.toFixed(4), + errorRate: (analytics.errorRate * 100).toFixed(2) + '%', + }, + providers: analytics.providerBreakdown, + costs: { + daily: CostTracker.getDailyCost().toFixed(4), + monthly: CostTracker.getMonthlyCost().toFixed(4), + breakdown: costBreakdown, + }, + alerts: await getActiveAlerts(), + }; + + return Response.json(dashboard); +} + +function parseTimeRange(range: string): number { + const unit = range.slice(-1); + const value = parseInt(range.slice(0, -1)); + + switch (unit) { + case 'h': return value * 60 * 60 * 1000; + case 'd': return value * 24 * 60 * 60 * 1000; + case 'w': return value * 7 * 24 * 60 * 60 * 1000; + default: return 60 * 60 * 1000; // Default to 1 hour + } +} + +async function getActiveAlerts() { + // Return active alerts from your alerting system + return []; +} +``` + +#### React Dashboard Component + +```typescript +// components/monitoring-dashboard.tsx +'use client'; + +import { useEffect, useState } from 'react'; +import { LineChart, Line, BarChart, Bar, PieChart, Pie, Cell, ResponsiveContainer, XAxis, YAxis, Tooltip, Legend } from 'recharts'; + +interface DashboardData { + overview: { + totalRequests: number; + averageLatency: number; + totalTokens: number; + totalCost: string; + errorRate: string; + }; + providers: Record<string, any>; + costs: { + daily: string; + monthly: string; + breakdown: Record<string, number>; + }; +} + +export default function MonitoringDashboard() { + const [data, setData] = useState<DashboardData | null>(null); + const [timeRange, setTimeRange] = useState('1h'); + const [loading, setLoading] = useState(true); + + useEffect(() => { + fetchDashboardData(); + const interval = setInterval(fetchDashboardData, 30000); // Refresh every 30 seconds + return () => clearInterval(interval); + }, [timeRange]); + + const fetchDashboardData = async () => { + try { + const response = await fetch(`/api/monitoring/dashboard?range=${timeRange}`); + const dashboardData = await response.json(); + setData(dashboardData); + } catch (error) { + console.error('Failed to fetch dashboard data:', error); + } finally { + setLoading(false); + } + }; + + if (loading) { + return <div className="flex justify-center items-center h-64">Loading dashboard...</div>; + } + + if (!data) { + return <div className="text-center text-red-500">Failed to load dashboard data</div>; + } + + const providerColors = ['#8884d8', '#82ca9d', '#ffc658', '#ff7c7c', '#8dd1e1']; + const costBreakdownData = Object.entries(data.costs.breakdown).map(([provider, cost]) => ({ + provider, + cost: parseFloat(cost.toFixed(4)), + })); + + return ( + <div className="p-6 max-w-7xl mx-auto"> + <div className="mb-6 flex justify-between items-center"> + <h1 className="text-3xl font-bold">AI SDK Monitoring Dashboard</h1> + <select + value={timeRange} + onChange={(e) => setTimeRange(e.target.value)} + className="border rounded px-3 py-2" + > + <option value="1h">Last Hour</option> + <option value="6h">Last 6 Hours</option> + <option value="1d">Last Day</option> + <option value="7d">Last Week</option> + </select> + </div> + + {/* Overview Cards */} + <div className="grid grid-cols-1 md:grid-cols-5 gap-6 mb-8"> + <div className="bg-white p-4 rounded-lg shadow"> + <h3 className="text-sm font-medium text-gray-500">Total Requests</h3> + <p className="text-2xl font-bold">{data.overview.totalRequests.toLocaleString()}</p> + </div> + <div className="bg-white p-4 rounded-lg shadow"> + <h3 className="text-sm font-medium text-gray-500">Avg Latency</h3> + <p className="text-2xl font-bold">{data.overview.averageLatency}ms</p> + </div> + <div className="bg-white p-4 rounded-lg shadow"> + <h3 className="text-sm font-medium text-gray-500">Total Tokens</h3> + <p className="text-2xl font-bold">{data.overview.totalTokens.toLocaleString()}</p> + </div> + <div className="bg-white p-4 rounded-lg shadow"> + <h3 className="text-sm font-medium text-gray-500">Total Cost</h3> + <p className="text-2xl font-bold">${data.overview.totalCost}</p> + </div> + <div className="bg-white p-4 rounded-lg shadow"> + <h3 className="text-sm font-medium text-gray-500">Error Rate</h3> + <p className="text-2xl font-bold text-red-500">{data.overview.errorRate}</p> + </div> + </div> + + {/* Charts */} + <div className="grid grid-cols-1 lg:grid-cols-2 gap-8"> + {/* Cost Breakdown */} + <div className="bg-white p-6 rounded-lg shadow"> + <h3 className="text-lg font-semibold mb-4">Cost Breakdown by Provider</h3> + <ResponsiveContainer width="100%" height={300}> + <PieChart> + <Pie + data={costBreakdownData} + dataKey="cost" + nameKey="provider" + cx="50%" + cy="50%" + outerRadius={100} + label={({ provider, cost }) => `${provider}: $${cost}`} + > + {costBreakdownData.map((entry, index) => ( + <Cell key={`cell-${index}`} fill={providerColors[index % providerColors.length]} /> + ))} + </Pie> + <Tooltip /> + </PieChart> + </ResponsiveContainer> + </div> + + {/* Provider Usage */} + <div className="bg-white p-6 rounded-lg shadow"> + <h3 className="text-lg font-semibold mb-4">Provider Usage</h3> + <ResponsiveContainer width="100%" height={300}> + <BarChart data={Object.entries(data.providers).map(([provider, stats]) => ({ + provider, + requests: stats.requests, + tokens: stats.tokens, + }))}> + <XAxis dataKey="provider" /> + <YAxis /> + <Tooltip /> + <Legend /> + <Bar dataKey="requests" fill="#8884d8" name="Requests" /> + <Bar dataKey="tokens" fill="#82ca9d" name="Tokens (thousands)" /> + </BarChart> + </ResponsiveContainer> + </div> + </div> + + {/* Cost Summary */} + <div className="mt-8 bg-white p-6 rounded-lg shadow"> + <h3 className="text-lg font-semibold mb-4">Cost Summary</h3> + <div className="grid grid-cols-1 md:grid-cols-2 gap-4"> + <div> + <p className="text-sm text-gray-500">Daily Cost</p> + <p className="text-xl font-bold">${data.costs.daily}</p> + </div> + <div> + <p className="text-sm text-gray-500">Monthly Cost</p> + <p className="text-xl font-bold">${data.costs.monthly}</p> + </div> + </div> + </div> + </div> + ); +} +``` + +### Integration with Existing Code + +- **API Routes**: Wrap with monitoring middleware automatically +- **Streaming**: Built-in performance tracking for streaming responses +- **Error Handling**: Automatic error classification and reporting +- **Cost Tracking**: Real-time cost calculation across all providers +- **Alerting**: Budget and performance threshold alerting +- **Dashboard**: Real-time monitoring dashboard with visualizations +- **External Services**: Integration with DataDog, New Relic, custom analytics + +Focus on building comprehensive monitoring that provides actionable insights for AI SDK applications while maintaining low overhead and high accuracy.
\ No newline at end of file diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-provider-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-provider-setup.md new file mode 100644 index 0000000..0d83374 --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-provider-setup.md @@ -0,0 +1,169 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Configure AI providers and multi-provider setup +argument-hint: "[single|multi|fallback] [anthropic|openai|google|cohere]" +--- + +## Set up AI Provider Configuration + +Configure robust AI provider setup with the Vercel AI SDK: $ARGUMENTS + +### Current Configuration Analysis + +Existing provider setup: !`grep -r "@ai-sdk/" . --include="*.ts" --include="*.tsx" | head -5` + +Environment variables: !`grep -r "API_KEY\|_KEY" .env* 2>/dev/null | head -5 || echo "No API keys found in .env files"` + +Provider imports: !`grep -r "from '@ai-sdk/" . --include="*.ts" | head -10` + +### Configuration Strategy + +**Single Provider**: Focus on one provider with optimal configuration +**Multi Provider**: Set up multiple providers with consistent interface +**Fallback**: Implement automatic failover between providers + +### Your Task + +1. **Analyze current provider setup** and identify improvements needed +2. **Design provider architecture** with proper abstraction layers +3. **Implement configuration management** with environment handling +4. **Set up provider fallback logic** for reliability +5. **Add usage tracking and cost monitoring** for optimization +6. **Create provider health checks** for monitoring +7. **Implement proper error handling** and recovery +8. **Add comprehensive testing** for all providers + +### Implementation Requirements + +#### Environment Configuration + +- Secure API key management +- Environment-specific configurations +- Provider availability detection +- Default provider selection +- Feature flag support for provider switching + +#### Provider Abstraction + +- Unified interface across all providers +- Model capability mapping +- Feature detection and adaptation +- Consistent error handling +- Performance monitoring integration + +#### Fallback and Reliability + +- Automatic provider failover +- Health check implementation +- Circuit breaker patterns +- Retry logic with exponential backoff +- Graceful degradation strategies + +### Expected Deliverables + +1. **Provider configuration system** with environment management +2. **Multi-provider client wrapper** with unified interface +3. **Fallback and health monitoring** implementation +4. **Usage tracking and analytics** system +5. **Cost optimization utilities** for model selection +6. **Testing suite** covering all provider scenarios +7. **Documentation** with setup guides and best practices + +### Provider-Specific Optimizations + +#### Anthropic Configuration + +- Claude model selection (Haiku, Sonnet, Opus, Claude 4) +- Extended thinking capabilities setup +- Prompt caching configuration +- Tool use optimization +- Context window management + +#### OpenAI Configuration + +- Model selection (GPT-3.5, GPT-4, GPT-4o, O1) +- Responses API integration +- Function calling optimization +- Structured output configuration +- Built-in tool integration (web search) + +#### Google Configuration + +- Gemini model variants setup +- Search grounding capabilities +- Multimodal processing optimization +- Safety settings configuration +- Thinking mode integration + +#### Cohere Configuration + +- Command model setup +- RAG optimization +- Embedding integration +- Multilingual support +- Custom model fine-tuning + +### Cost Management + +#### Usage Tracking + +- Token usage monitoring across providers +- Cost calculation and reporting +- Budget limits and alerting +- Usage pattern analysis +- Optimization recommendations + +#### Model Selection + +- Intelligent model routing based on task complexity +- Cost-performance optimization +- Usage-based provider selection +- Dynamic model switching +- A/B testing for provider performance + +### Security and Compliance + +#### API Key Management + +- Secure key storage and rotation +- Environment variable validation +- Access control and permissions +- Audit logging for API usage +- Compliance reporting + +#### Data Privacy + +- Request/response logging controls +- Data retention policies +- Regional data handling +- Privacy-preserving configurations +- Compliance monitoring + +### Monitoring and Observability + +#### Health Monitoring + +- Provider availability checks +- Latency and performance monitoring +- Error rate tracking +- Success rate analysis +- Capacity utilization monitoring + +#### Analytics and Reporting + +- Usage dashboards and reports +- Cost analysis and forecasting +- Performance benchmarking +- User behavior analysis +- Provider comparison metrics + +### Testing Strategy + +- Provider connectivity tests +- Failover scenario testing +- Performance and load testing +- Cost calculation validation +- Security and compliance testing +- Integration testing with applications + +Focus on building a robust, cost-effective, and reliable multi-provider architecture that ensures high availability and optimal performance while maintaining security and compliance standards. diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-rag-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-rag-setup.md new file mode 100644 index 0000000..5003af9 --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-rag-setup.md @@ -0,0 +1,252 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Set up RAG (Retrieval-Augmented Generation) system +argument-hint: "[basic|advanced|conversational|agentic]" +--- + +## Set up RAG (Retrieval-Augmented Generation) System + +Create a comprehensive RAG implementation with embeddings, vector storage, and retrieval: $ARGUMENTS + +### Current Project Analysis + +Existing database setup: !`find . -name "*schema*" -o -name "*migration*" -o -name "drizzle.config.*" | head -5` + +Vector database configuration: !`grep -r "vector\|embedding" . --include="*.ts" --include="*.sql" | head -5` + +AI SDK integration: !`grep -r "embed\|embedMany" . --include="*.ts" | head -5` + +### RAG Implementation Types + +**Basic RAG**: Simple query → retrieve → generate pipeline +**Advanced RAG**: Multi-query, re-ranking, hybrid search, filtering +**Conversational RAG**: Context-aware retrieval with chat history +**Agentic RAG**: Tool-based retrieval with dynamic knowledge access + +### Your Task + +1. **Analyze current data infrastructure** and vector storage capabilities +2. **Design embedding and chunking strategy** for optimal retrieval +3. **Set up vector database** with proper indexing and search +4. **Implement embedding pipeline** with batch processing +5. **Create retrieval system** with similarity search and ranking +6. **Build RAG generation pipeline** with context injection +7. **Add evaluation metrics** for retrieval and generation quality +8. **Implement comprehensive testing** for all RAG components + +### Implementation Requirements + +#### Data Processing Pipeline + +- Document ingestion and preprocessing +- Intelligent chunking strategies (sentence, semantic, sliding window) +- Metadata extraction and enrichment +- Batch embedding generation with rate limiting +- Deduplication and quality filtering + +#### Vector Storage and Search + +- Database setup (PostgreSQL + pgvector, Pinecone, Supabase, etc.) +- Proper indexing (HNSW, IVFFlat) for performance +- Similarity search with filtering and ranking +- Hybrid search combining vector and text search +- Metadata filtering and faceted search + +#### RAG Generation + +- Context selection and ranking +- Prompt engineering for RAG scenarios +- Context window management +- Response grounding and source attribution +- Quality control and relevance scoring + +### Expected Deliverables + +1. **Document processing pipeline** with chunking and embedding +2. **Vector database setup** with optimized indexing +3. **Retrieval system** with advanced search capabilities +4. **RAG generation API** with streaming support +5. **Evaluation framework** for quality measurement +6. **Admin interface** for content management +7. **Comprehensive documentation** and examples + +### Database Schema Design + +#### PostgreSQL with pgvector + +```sql +-- Enable vector extension +CREATE EXTENSION IF NOT EXISTS vector; + +-- Documents table +CREATE TABLE documents ( + id SERIAL PRIMARY KEY, + title VARCHAR(255), + content TEXT NOT NULL, + metadata JSONB, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +-- Chunks table +CREATE TABLE document_chunks ( + id SERIAL PRIMARY KEY, + document_id INTEGER REFERENCES documents(id) ON DELETE CASCADE, + content TEXT NOT NULL, + chunk_index INTEGER, + metadata JSONB, + embedding VECTOR(1536), + created_at TIMESTAMP DEFAULT NOW() +); + +-- Indexes for performance +CREATE INDEX ON document_chunks USING hnsw (embedding vector_cosine_ops); +CREATE INDEX ON document_chunks (document_id); +CREATE INDEX ON documents USING gin (metadata); +``` + +#### Drizzle ORM Schema + +```typescript +export const documents = pgTable('documents', { + id: serial('id').primaryKey(), + title: varchar('title', { length: 255 }), + content: text('content').notNull(), + metadata: jsonb('metadata'), + createdAt: timestamp('created_at').defaultNow(), + updatedAt: timestamp('updated_at').defaultNow(), +}); + +export const documentChunks = pgTable( + 'document_chunks', + { + id: serial('id').primaryKey(), + documentId: integer('document_id').references(() => documents.id, { + onDelete: 'cascade', + }), + content: text('content').notNull(), + chunkIndex: integer('chunk_index'), + metadata: jsonb('metadata'), + embedding: vector('embedding', { dimensions: 1536 }), + createdAt: timestamp('created_at').defaultNow(), + }, + (table) => ({ + embeddingIndex: index('embedding_idx').using( + 'hnsw', + table.embedding.op('vector_cosine_ops'), + ), + documentIdIndex: index('document_id_idx').on(table.documentId), + }), +); +``` + +### Embedding Strategy + +#### Chunking Algorithms + +- **Sentence-based**: Split on sentence boundaries for coherent chunks +- **Semantic**: Use NLP models to identify semantic boundaries +- **Sliding window**: Overlapping chunks to preserve context +- **Recursive**: Hierarchical chunking for different granularities + +#### Model Selection + +- **OpenAI**: text-embedding-3-small/large for versatility +- **Cohere**: embed-english-v3.0 for specialized domains +- **Local models**: Sentence-transformers for privacy/cost +- **Multilingual**: Support for multiple languages + +### Advanced RAG Patterns + +#### Multi-Query RAG + +```typescript +async function multiQueryRAG(userQuery: string) { + // Generate multiple query variants + const queryVariants = await generateQueryVariants(userQuery); + + // Retrieve for each variant + const retrievalResults = await Promise.all( + queryVariants.map(query => retrieveDocuments(query)) + ); + + // Combine and re-rank results + const combinedResults = combineAndRerankResults(retrievalResults); + + return combinedResults; +} +``` + +#### Conversational RAG + +```typescript +async function conversationalRAG(messages: Message[], query: string) { + // Extract conversation context + const conversationContext = extractContext(messages); + + // Generate context-aware query + const contextualQuery = await generateContextualQuery(query, conversationContext); + + // Retrieve with conversation awareness + const documents = await retrieveWithContext(contextualQuery, conversationContext); + + return documents; +} +``` + +### Quality Evaluation + +#### Retrieval Metrics + +- **Precision@K**: Relevant documents in top-K results +- **Recall@K**: Coverage of relevant documents +- **MRR**: Mean Reciprocal Rank of first relevant document +- **NDCG**: Normalized Discounted Cumulative Gain + +#### Generation Metrics + +- **Faithfulness**: Response grounded in retrieved context +- **Relevance**: Response relevance to user query +- **Completeness**: Coverage of important information +- **Coherence**: Logical flow and readability + +### Testing and Validation + +#### Unit Testing + +- Embedding generation accuracy +- Chunking algorithm correctness +- Similarity search precision +- Database operations integrity + +#### Integration Testing + +- End-to-end RAG pipeline +- Performance under load +- Quality with various document types +- Scalability testing + +#### Evaluation Testing + +- Golden dataset evaluation +- A/B testing with different strategies +- User feedback collection +- Continuous quality monitoring + +### Performance Optimization + +#### Database Optimization + +- Proper indexing strategies (HNSW vs IVFFlat) +- Connection pooling and caching +- Query optimization and profiling +- Horizontal scaling considerations + +#### Embedding Optimization + +- Batch processing for efficiency +- Caching frequently used embeddings +- Model quantization for speed +- Parallel processing pipelines + +Focus on building a production-ready RAG system that provides accurate, relevant, and fast retrieval-augmented generation with proper evaluation and optimization strategies. diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-streaming-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-streaming-setup.md new file mode 100644 index 0000000..0d65349 --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-streaming-setup.md @@ -0,0 +1,82 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Set up streaming AI responses with proper error handling +argument-hint: "[text|object|chat|completion]" +--- + +## Set up AI Streaming Implementation + +Create a robust streaming AI implementation with the Vercel AI SDK for: $ARGUMENTS + +### Current Project Analysis + +Project structure: !`find . -type f -name "*.ts" -o -name "*.tsx" | grep -E "(api|components|lib)" | head -10` + +Existing AI SDK setup: !`grep -r "from 'ai'" . --include="*.ts" --include="*.tsx" | head -5` + +Package dependencies: !`cat package.json | jq '.dependencies | to_entries[] | select(.key | contains("ai")) | "\(.key): \(.value)"' -r 2>/dev/null || echo "No AI dependencies found"` + +### Streaming Type Analysis + +**Text Streaming**: Real-time text generation with token-by-token updates +**Object Streaming**: Structured data streaming with partial object updates +**Chat Streaming**: Conversational interfaces with message history +**Completion Streaming**: Single-turn completions with progressive updates + +### Your Task + +1. **Assess current streaming setup** and identify gaps +2. **Implement the appropriate streaming pattern** based on the specified type +3. **Create robust error handling** for stream interruptions and failures +4. **Add proper loading states** and user feedback +5. **Implement stream cancellation** for better UX +6. **Set up proper TypeScript types** for streaming responses +7. **Add performance optimizations** (chunking, backpressure handling) +8. **Include comprehensive testing** for edge cases + +### Implementation Requirements + +#### Server-Side Streaming + +- Proper route handler setup with `maxDuration` +- Model configuration with appropriate parameters +- Stream response formatting with `toUIMessageStreamResponse()` or `toTextStreamResponse()` +- Abort signal handling for stream cancellation +- Error boundaries and fallback responses + +#### Client-Side Streaming + +- React hooks for stream management (`useChat`, `useCompletion`, `useObject`) +- Progressive UI updates with optimistic rendering +- Loading states and stream status indicators +- Error handling with retry mechanisms +- Stream interruption and cancellation + +#### Performance Considerations + +- Appropriate chunk sizing for smooth updates +- Memory management for long streams +- Connection pooling and reuse +- Backpressure handling for slow consumers +- Optimization for mobile and slow connections + +### Expected Deliverables + +1. **Streaming API route** with proper configuration +2. **React streaming component** with modern patterns +3. **TypeScript interfaces** for streaming data +4. **Error handling implementation** with recovery +5. **Performance optimizations** for production +6. **Testing suite** for streaming functionality +7. **Documentation** with usage examples + +### Testing Requirements + +- Stream start and completion scenarios +- Network interruption and recovery +- Concurrent stream handling +- Error conditions and fallbacks +- Performance under load +- Mobile and slow connection testing + +Focus on building a production-ready streaming implementation that provides excellent user experience with proper error handling and performance optimization. diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-tools-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-tools-setup.md new file mode 100644 index 0000000..9e33b6f --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-tools-setup.md @@ -0,0 +1,137 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Create AI tools and function calling capabilities +argument-hint: "[simple|database|api|multimodal|agent]" +--- + +## Set up AI Tools and Function Calling + +Create comprehensive AI tool integrations with the Vercel AI SDK for: $ARGUMENTS + +### Current Project Analysis + +Existing tool implementations: !`grep -r "import.*tool" . --include="*.ts" --include="*.tsx" | head -5` + +API integrations: !`grep -r "fetch\|axios" . --include="*.ts" | head -5` + +Database setup: !`find . -name "*schema*" -o -name "*db*" -o -name "*database*" | grep -v node_modules | head -5` + +### Tool Type Requirements + +**Simple Tools**: Basic utility functions (calculator, formatter, validator) +**Database Tools**: Safe database queries, data retrieval, analytics +**API Tools**: External service integrations, webhooks, data fetching +**Multimodal Tools**: Image processing, document analysis, file handling +**Agent Tools**: Complex workflows, multi-step operations, decision making + +### Your Task + +1. **Analyze the project needs** and identify appropriate tool types +2. **Design tool schemas** with proper Zod validation +3. **Implement secure execution logic** with error handling +4. **Set up proper authentication** and authorization +5. **Add comprehensive input validation** and sanitization +6. **Implement rate limiting** and usage monitoring +7. **Create tool testing suite** for reliability +8. **Document tool usage** and examples + +### Implementation Guidelines + +#### Tool Definition Patterns + +```typescript +// Basic tool structure +const toolName = tool({ + description: 'Clear description of what the tool does', + inputSchema: z.object({ + param: z.string().describe('Parameter description'), + }), + execute: async ({ param }) => { + // Implementation with proper error handling + try { + const result = await performOperation(param); + return { success: true, data: result }; + } catch (error) { + return { success: false, error: error.message }; + } + }, +}); +``` + +#### Security Considerations + +- Input validation and sanitization +- Authentication and authorization checks +- Rate limiting and abuse prevention +- Secure API key management +- Output filtering and validation +- Audit logging for sensitive operations + +#### Error Handling + +- Graceful failure modes +- Informative error messages +- Retry mechanisms for transient failures +- Fallback strategies +- Circuit breaker patterns +- Monitoring and alerting + +### Expected Deliverables + +1. **Tool definitions** with proper schemas and validation +2. **Execution implementations** with robust error handling +3. **Agent integration** with multi-step capabilities +4. **Security middleware** for authentication and rate limiting +5. **Testing suite** covering all tool scenarios +6. **Usage analytics** and monitoring +7. **Documentation** with examples and best practices + +### Tool Categories to Implement + +#### Data & Analytics Tools + +- Database query execution +- Data aggregation and analysis +- Report generation +- Chart and visualization creation + +#### External Integration Tools + +- REST API clients +- Webhook handlers +- File processing and storage +- Email and notification services + +#### Utility Tools + +- Text processing and formatting +- Mathematical calculations +- Data validation and transformation +- Code generation and analysis + +#### Advanced Agent Tools + +- Multi-step workflow orchestration +- Decision tree navigation +- Dynamic tool selection +- Context-aware processing + +### Testing Requirements + +- Unit tests for each tool execution path +- Integration tests with external services +- Security tests for input validation +- Performance tests under load +- Error scenario testing +- End-to-end agent workflow tests + +### Monitoring and Observability + +- Tool usage metrics and analytics +- Performance monitoring and latency tracking +- Error rate monitoring and alerting +- Cost tracking for external API usage +- Security audit logging +- User behavior analysis + +Focus on building secure, reliable, and well-tested tool integrations that enhance AI capabilities while maintaining proper security and monitoring practices. |