diff options
Diffstat (limited to 'tooling/vercel-ai-sdk/.claude/commands/ai-experimental-setup.md')
| -rw-r--r-- | tooling/vercel-ai-sdk/.claude/commands/ai-experimental-setup.md | 793 |
1 files changed, 793 insertions, 0 deletions
diff --git a/tooling/vercel-ai-sdk/.claude/commands/ai-experimental-setup.md b/tooling/vercel-ai-sdk/.claude/commands/ai-experimental-setup.md new file mode 100644 index 0000000..80d83ff --- /dev/null +++ b/tooling/vercel-ai-sdk/.claude/commands/ai-experimental-setup.md @@ -0,0 +1,793 @@ +--- +allowed-tools: Read, Write, Edit, MultiEdit, Bash +description: Enable cutting-edge experimental AI SDK features safely +argument-hint: "[beta|experimental|research|custom]" +--- + +## Enable Experimental AI SDK Features + +Configure and safely enable cutting-edge AI SDK experimental features: $ARGUMENTS + +### Current Experimental Status + +Existing experimental features: !`grep -r "experimental\|beta\|alpha" . --include="*.ts" --include="*.json" | head -5` + +Feature flags: !`grep -r "ENABLE_\|FLAG_" .env* 2>/dev/null | head -3 || echo "No feature flags found"` + +Advanced configurations: !`grep -r "streamingTimeouts\|thinkingMode\|toolCallStreaming" . --include="*.ts" | head -5` + +### Experimental Feature Categories + +**Beta Features**: Stable experimental features ready for production testing +**Experimental**: Cutting-edge features in active development +**Research**: Bleeding-edge research features for experimentation +**Custom**: Custom experimental implementations and modifications + +### Your Task + +1. **Analyze experimental feature landscape** and identify safe options +2. **Implement feature flag system** for controlled rollouts +3. **Configure experimental AI SDK options** with proper safeguards +4. **Set up A/B testing framework** for feature validation +5. **Add monitoring and telemetry** for experimental features +6. **Create fallback mechanisms** for experimental feature failures +7. **Implement gradual rollout strategy** with user controls +8. **Add comprehensive testing** for experimental features + +### Implementation Requirements + +#### Feature Flag System + +- Environment-based feature control +- User-level feature toggles +- Percentage-based rollouts +- Real-time feature flag updates +- Fallback mechanisms for failures + +#### Safety Measures + +- Automatic fallback to stable features +- Error isolation and reporting +- Performance impact monitoring +- User experience protection +- Data integrity guarantees + +#### Experimental Configuration + +- Advanced streaming options +- Cutting-edge model features +- Research-level AI capabilities +- Custom provider integrations +- Performance optimizations + +### Expected Deliverables + +1. **Feature flag system** with environment and user controls +2. **Experimental AI SDK configurations** with safety controls +3. **A/B testing framework** for feature validation +4. **Monitoring and telemetry** for experimental features +5. **Fallback mechanisms** for reliability +6. **Documentation** for experimental feature usage +7. **Testing suite** covering experimental scenarios + +### Feature Flag Infrastructure + +#### Core Feature Flag System + +```typescript +// lib/experimental/feature-flags.ts +interface FeatureFlag { + name: string; + enabled: boolean; + rolloutPercentage: number; + conditions?: { + userIds?: string[]; + environments?: string[]; + regions?: string[]; + custom?: (context: any) => boolean; + }; + metadata?: { + description: string; + added: string; + owner: string; + stableDate?: string; + }; +} + +export class ExperimentalFeatureManager { + private static instance: ExperimentalFeatureManager; + private flags: Map<string, FeatureFlag> = new Map(); + private context: any = {}; + + static getInstance(): ExperimentalFeatureManager { + if (!ExperimentalFeatureManager.instance) { + ExperimentalFeatureManager.instance = new ExperimentalFeatureManager(); + } + return ExperimentalFeatureManager.instance; + } + + async initialize(context: any = {}) { + this.context = context; + await this.loadFeatureFlags(); + } + + private async loadFeatureFlags() { + // Load from environment variables + const envFlags = this.loadFromEnvironment(); + + // Load from external service (optional) + const remoteFlags = await this.loadFromRemoteService(); + + // Merge flags with priority: remote > environment > defaults + const allFlags = { ...this.getDefaultFlags(), ...envFlags, ...remoteFlags }; + + Object.entries(allFlags).forEach(([name, flag]) => { + this.flags.set(name, flag as FeatureFlag); + }); + } + + private getDefaultFlags(): Record<string, FeatureFlag> { + return { + 'reasoning-models': { + name: 'reasoning-models', + enabled: false, + rolloutPercentage: 0, + metadata: { + description: 'Enable O1, O3-mini, and DeepSeek reasoning models', + added: '2024-12-01', + owner: 'ai-team', + }, + }, + 'computer-use': { + name: 'computer-use', + enabled: false, + rolloutPercentage: 0, + conditions: { + environments: ['development', 'staging'], + }, + metadata: { + description: 'Enable Claude 3.5 Sonnet computer use capabilities', + added: '2024-12-01', + owner: 'automation-team', + }, + }, + 'generative-ui': { + name: 'generative-ui', + enabled: true, + rolloutPercentage: 100, + metadata: { + description: 'Enable streamUI for dynamic component generation', + added: '2024-11-01', + owner: 'ui-team', + }, + }, + 'advanced-streaming': { + name: 'advanced-streaming', + enabled: true, + rolloutPercentage: 50, + metadata: { + description: 'Advanced streaming patterns with multi-step and waitUntil', + added: '2024-11-15', + owner: 'streaming-team', + }, + }, + 'edge-optimization': { + name: 'edge-optimization', + enabled: true, + rolloutPercentage: 75, + conditions: { + environments: ['production', 'staging'], + }, + metadata: { + description: 'Vercel Edge Runtime optimizations', + added: '2024-10-01', + owner: 'performance-team', + }, + }, + 'natural-language-sql': { + name: 'natural-language-sql', + enabled: false, + rolloutPercentage: 25, + conditions: { + custom: (context) => context.hasDatabase === true, + }, + metadata: { + description: 'Natural language to SQL conversion', + added: '2024-12-10', + owner: 'data-team', + }, + }, + }; + } + + private loadFromEnvironment(): Record<string, Partial<FeatureFlag>> { + const flags: Record<string, Partial<FeatureFlag>> = {}; + + // Load from environment variables + if (process.env.ENABLE_REASONING_MODELS === 'true') { + flags['reasoning-models'] = { enabled: true, rolloutPercentage: 100 }; + } + + if (process.env.ENABLE_COMPUTER_USE === 'true') { + flags['computer-use'] = { enabled: true, rolloutPercentage: 100 }; + } + + if (process.env.ENABLE_GENERATIVE_UI === 'true') { + flags['generative-ui'] = { enabled: true, rolloutPercentage: 100 }; + } + + if (process.env.ENABLE_ADVANCED_STREAMING === 'true') { + flags['advanced-streaming'] = { enabled: true, rolloutPercentage: 100 }; + } + + if (process.env.ENABLE_EDGE_OPTIMIZATION === 'true') { + flags['edge-optimization'] = { enabled: true, rolloutPercentage: 100 }; + } + + return flags; + } + + private async loadFromRemoteService(): Promise<Record<string, Partial<FeatureFlag>>> { + // Optional: Load from external feature flag service + try { + if (process.env.FEATURE_FLAG_SERVICE_URL) { + const response = await fetch(process.env.FEATURE_FLAG_SERVICE_URL, { + headers: { + 'Authorization': `Bearer ${process.env.FEATURE_FLAG_API_KEY}`, + }, + }); + + if (response.ok) { + return await response.json(); + } + } + } catch (error) { + console.warn('Failed to load remote feature flags:', error); + } + + return {}; + } + + isEnabled(flagName: string, userId?: string): boolean { + const flag = this.flags.get(flagName); + if (!flag) return false; + + // Check basic enabled status + if (!flag.enabled) return false; + + // Check conditions + if (flag.conditions) { + if (flag.conditions.userIds && userId) { + if (!flag.conditions.userIds.includes(userId)) return false; + } + + if (flag.conditions.environments) { + const env = process.env.NODE_ENV || 'development'; + if (!flag.conditions.environments.includes(env)) return false; + } + + if (flag.conditions.regions) { + const region = process.env.VERCEL_REGION || 'local'; + if (!flag.conditions.regions.includes(region)) return false; + } + + if (flag.conditions.custom) { + if (!flag.conditions.custom(this.context)) return false; + } + } + + // Check rollout percentage + if (flag.rolloutPercentage < 100) { + const hash = this.getUserHash(userId || 'anonymous', flagName); + if (hash % 100 >= flag.rolloutPercentage) return false; + } + + return true; + } + + private getUserHash(userId: string, flagName: string): number { + // Simple hash function for consistent user bucketing + let hash = 0; + const str = `${userId}-${flagName}`; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32-bit integer + } + return Math.abs(hash); + } + + getAllFlags(): Map<string, FeatureFlag> { + return new Map(this.flags); + } + + updateFlag(flagName: string, updates: Partial<FeatureFlag>) { + const existing = this.flags.get(flagName); + if (existing) { + this.flags.set(flagName, { ...existing, ...updates }); + } + } + + async trackFeatureUsage(flagName: string, userId?: string, metadata?: any) { + const usage = { + flag: flagName, + userId, + timestamp: Date.now(), + context: this.context, + metadata, + }; + + // Send to analytics service + await this.sendUsageToAnalytics(usage); + } + + private async sendUsageToAnalytics(usage: any) { + try { + if (process.env.ANALYTICS_ENDPOINT) { + await fetch(process.env.ANALYTICS_ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(usage), + }); + } + } catch (error) { + console.warn('Failed to send feature usage analytics:', error); + } + } +} + +// Singleton instance +export const featureFlags = ExperimentalFeatureManager.getInstance(); +``` + +#### Experimental AI SDK Wrapper + +```typescript +// lib/experimental/ai-sdk-experimental.ts +import { streamText, generateText, streamUI, generateObject } from 'ai'; +import { featureFlags } from './feature-flags'; + +export interface ExperimentalOptions { + userId?: string; + fallbackOnError?: boolean; + trackUsage?: boolean; +} + +export class ExperimentalAISDK { + + static async streamText(config: any, options: ExperimentalOptions = {}) { + const { userId, fallbackOnError = true, trackUsage = true } = options; + + // Apply experimental features based on flags + const experimentalConfig = await this.applyExperimentalFeatures(config, userId); + + try { + const result = streamText(experimentalConfig); + + if (trackUsage) { + await this.trackExperimentalUsage(experimentalConfig, userId); + } + + return result; + } catch (error) { + if (fallbackOnError) { + console.warn('Experimental feature failed, falling back to stable:', error); + return streamText(config); // Fallback to original config + } + throw error; + } + } + + static async generateText(config: any, options: ExperimentalOptions = {}) { + const { userId, fallbackOnError = true, trackUsage = true } = options; + + const experimentalConfig = await this.applyExperimentalFeatures(config, userId); + + try { + const result = await generateText(experimentalConfig); + + if (trackUsage) { + await this.trackExperimentalUsage(experimentalConfig, userId); + } + + return result; + } catch (error) { + if (fallbackOnError) { + console.warn('Experimental feature failed, falling back to stable:', error); + return generateText(config); + } + throw error; + } + } + + static async streamUI(config: any, options: ExperimentalOptions = {}) { + const { userId, fallbackOnError = true, trackUsage = true } = options; + + if (!featureFlags.isEnabled('generative-ui', userId)) { + throw new Error('Generative UI is not enabled for this user'); + } + + try { + const result = streamUI(config); + + if (trackUsage) { + await featureFlags.trackFeatureUsage('generative-ui', userId, { + toolCount: Object.keys(config.tools || {}).length, + }); + } + + return result; + } catch (error) { + if (fallbackOnError) { + // Fallback to regular text streaming + console.warn('StreamUI failed, falling back to streamText:', error); + return streamText({ + ...config, + text: ({ content }) => content, // Simple text output + }); + } + throw error; + } + } + + private static async applyExperimentalFeatures(config: any, userId?: string) { + const experimentalConfig = { ...config }; + + // Advanced streaming features + if (featureFlags.isEnabled('advanced-streaming', userId)) { + experimentalConfig.experimental_streamingTimeouts = { + streamingTimeout: 45000, + completeTimeout: 120000, + keepAliveInterval: 5000, + }; + + experimentalConfig.experimental_toolCallStreaming = true; + experimentalConfig.experimental_continueSteps = true; + + await featureFlags.trackFeatureUsage('advanced-streaming', userId); + } + + // Reasoning models + if (featureFlags.isEnabled('reasoning-models', userId)) { + if (config.model?.includes?.('o1') || config.model?.includes?.('reasoner')) { + experimentalConfig.experimental_reasoning = true; + experimentalConfig.experimental_thinkingMode = 'visible'; + experimentalConfig.experimental_thinkingTokens = true; + + await featureFlags.trackFeatureUsage('reasoning-models', userId); + } + } + + // Edge optimizations + if (featureFlags.isEnabled('edge-optimization', userId)) { + experimentalConfig.experimental_edgeOptimization = { + enableCompression: true, + enableKeepAlive: true, + connectionPooling: true, + }; + + experimentalConfig.experimental_memoryManagement = { + maxTokensInMemory: 25000, + enableGarbageCollection: true, + cleanupInterval: 30000, + }; + + await featureFlags.trackFeatureUsage('edge-optimization', userId); + } + + return experimentalConfig; + } + + private static async trackExperimentalUsage(config: any, userId?: string) { + const experimentalFeatures = []; + + if (config.experimental_streamingTimeouts) { + experimentalFeatures.push('advanced-streaming'); + } + + if (config.experimental_reasoning) { + experimentalFeatures.push('reasoning-models'); + } + + if (config.experimental_edgeOptimization) { + experimentalFeatures.push('edge-optimization'); + } + + for (const feature of experimentalFeatures) { + await featureFlags.trackFeatureUsage(feature, userId, { + configuration: Object.keys(config).filter(k => k.startsWith('experimental_')), + }); + } + } +} +``` + +### A/B Testing Framework + +#### Experiment Configuration + +```typescript +// lib/experimental/ab-testing.ts +export interface Experiment { + id: string; + name: string; + description: string; + status: 'draft' | 'running' | 'paused' | 'completed'; + variants: { + id: string; + name: string; + percentage: number; + config: any; + }[]; + targetAudience?: { + userIds?: string[]; + percentage?: number; + conditions?: any; + }; + metrics: string[]; + startDate: Date; + endDate?: Date; +} + +export class ExperimentManager { + private static instance: ExperimentManager; + private experiments: Map<string, Experiment> = new Map(); + + static getInstance(): ExperimentManager { + if (!ExperimentManager.instance) { + ExperimentManager.instance = new ExperimentManager(); + } + return ExperimentManager.instance; + } + + async initialize() { + await this.loadExperiments(); + } + + private async loadExperiments() { + // Load experiments from configuration + const defaultExperiments: Experiment[] = [ + { + id: 'reasoning-vs-standard', + name: 'Reasoning Models vs Standard Models', + description: 'Compare performance of O1 reasoning models vs standard models', + status: 'running', + variants: [ + { id: 'control', name: 'Standard Model', percentage: 50, config: { useReasoning: false } }, + { id: 'treatment', name: 'Reasoning Model', percentage: 50, config: { useReasoning: true } }, + ], + targetAudience: { percentage: 10 }, + metrics: ['response_quality', 'latency', 'cost', 'user_satisfaction'], + startDate: new Date('2024-12-01'), + endDate: new Date('2024-12-31'), + }, + { + id: 'streaming-optimization', + name: 'Advanced Streaming vs Basic Streaming', + description: 'Test advanced streaming features vs basic streaming', + status: 'running', + variants: [ + { id: 'control', name: 'Basic Streaming', percentage: 70, config: { advancedStreaming: false } }, + { id: 'treatment', name: 'Advanced Streaming', percentage: 30, config: { advancedStreaming: true } }, + ], + metrics: ['latency', 'error_rate', 'user_engagement'], + startDate: new Date('2024-11-15'), + endDate: new Date('2024-12-15'), + }, + ]; + + defaultExperiments.forEach(exp => { + this.experiments.set(exp.id, exp); + }); + } + + getVariant(experimentId: string, userId: string): any { + const experiment = this.experiments.get(experimentId); + if (!experiment || experiment.status !== 'running') { + return null; + } + + // Check if user is in target audience + if (!this.isUserInAudience(experiment, userId)) { + return null; + } + + // Determine variant based on user hash + const hash = this.getUserHash(userId, experimentId); + let cumulativePercentage = 0; + + for (const variant of experiment.variants) { + cumulativePercentage += variant.percentage; + if (hash % 100 < cumulativePercentage) { + return variant; + } + } + + return experiment.variants[0]; // Fallback to first variant + } + + private isUserInAudience(experiment: Experiment, userId: string): boolean { + if (!experiment.targetAudience) return true; + + if (experiment.targetAudience.userIds) { + return experiment.targetAudience.userIds.includes(userId); + } + + if (experiment.targetAudience.percentage) { + const hash = this.getUserHash(userId, experiment.id); + return (hash % 100) < experiment.targetAudience.percentage; + } + + return true; + } + + private getUserHash(userId: string, experimentId: string): number { + let hash = 0; + const str = `${userId}-${experimentId}`; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; + } + return Math.abs(hash); + } + + async recordMetric(experimentId: string, userId: string, metric: string, value: number) { + const variant = this.getVariant(experimentId, userId); + if (!variant) return; + + const record = { + experimentId, + variantId: variant.id, + userId, + metric, + value, + timestamp: Date.now(), + }; + + await this.sendMetricToAnalytics(record); + } + + private async sendMetricToAnalytics(record: any) { + try { + if (process.env.EXPERIMENT_ANALYTICS_ENDPOINT) { + await fetch(process.env.EXPERIMENT_ANALYTICS_ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(record), + }); + } + } catch (error) { + console.warn('Failed to send experiment metric:', error); + } + } +} + +export const experiments = ExperimentManager.getInstance(); +``` + +### API Integration + +#### Experimental API Route + +```typescript +// app/api/experimental/chat/route.ts +import { ExperimentalAISDK } from '@/lib/experimental/ai-sdk-experimental'; +import { experiments } from '@/lib/experimental/ab-testing'; +import { anthropic } from '@ai-sdk/anthropic'; + +export const runtime = 'edge'; +export const maxDuration = 300; + +export async function POST(req: Request) { + const { messages, userId } = await req.json(); + + try { + // Get experiment variant + const reasoningExperiment = experiments.getVariant('reasoning-vs-standard', userId); + const streamingExperiment = experiments.getVariant('streaming-optimization', userId); + + // Configure based on experiments + const config = { + model: reasoningExperiment?.config.useReasoning + ? anthropic('claude-3-sonnet-20240229') // Would use O1 in real implementation + : anthropic('claude-3-sonnet-20240229'), + messages, + }; + + // Use experimental SDK + const result = await ExperimentalAISDK.streamText(config, { + userId, + fallbackOnError: true, + trackUsage: true, + }); + + // Record experiment metrics + if (reasoningExperiment) { + // This would be implemented with actual metrics + await experiments.recordMetric('reasoning-vs-standard', userId, 'request_count', 1); + } + + return result.toUIMessageStreamResponse(); + + } catch (error) { + console.error('Experimental chat error:', error); + + // Fallback to stable implementation + const result = await ExperimentalAISDK.streamText({ + model: anthropic('claude-3-sonnet-20240229'), + messages, + }, { userId, fallbackOnError: false }); + + return result.toUIMessageStreamResponse(); + } +} +``` + +### Monitoring and Safety + +#### Experimental Feature Monitoring + +```typescript +// lib/experimental/monitoring.ts +export class ExperimentalMonitoring { + static async recordFeaturePerformance( + featureName: string, + metrics: { + latency?: number; + errorRate?: number; + userSatisfaction?: number; + cost?: number; + }, + userId?: string + ) { + const record = { + feature: featureName, + metrics, + userId, + timestamp: Date.now(), + environment: process.env.NODE_ENV, + version: process.env.APP_VERSION || 'unknown', + }; + + await this.sendToMonitoring(record); + } + + static async checkFeatureHealth(featureName: string): Promise<boolean> { + // Check if feature is performing within acceptable parameters + const healthData = await this.getFeatureHealthData(featureName); + + return ( + healthData.errorRate < 0.05 && // Less than 5% error rate + healthData.latency < 5000 && // Less than 5 second latency + healthData.userSatisfaction > 0.7 // Above 70% satisfaction + ); + } + + private static async sendToMonitoring(record: any) { + // Send to monitoring service + console.log('Experimental Feature Metrics:', record); + } + + private static async getFeatureHealthData(featureName: string) { + // This would fetch real health data from monitoring service + return { + errorRate: Math.random() * 0.1, + latency: Math.random() * 3000, + userSatisfaction: 0.7 + (Math.random() * 0.3), + }; + } +} +``` + +### Usage and Safety Guidelines + +- **Feature flags**: Control experimental features with environment variables and remote configuration +- **A/B testing**: Run controlled experiments to validate new features +- **Fallback mechanisms**: Always provide stable alternatives for experimental features +- **Monitoring**: Track performance and user experience of experimental features +- **Gradual rollout**: Start with small percentages and increase based on performance +- **Safety checks**: Implement automatic feature disabling for poor performance +- **Documentation**: Document experimental features and their expected behavior + +Focus on building a robust experimental framework that enables safe innovation while protecting user experience and system stability.
\ No newline at end of file |