import db from '../config/database'; import { logger } from '../utils/logger'; export interface ABTestAnalysisInput { tenantId: string; shopId: string; taskId: string; traceId: string; businessType: 'TOC' | 'TOB'; testId: string; analysisType: 'CONVERSION' | 'REVENUE' | 'CTR' | 'BOUNCE_RATE' | 'CUSTOM'; confidenceLevel?: number; minSampleSize?: number; } export interface ABTestAnalysisResult { success: boolean; testId: string; analysisType: string; analyzedAt: Date; variants: VariantAnalysis[]; winner?: { variantId: string; variantName: string; improvement: number; confidence: number; }; recommendation: string; isStatisticallySignificant: boolean; } export interface VariantAnalysis { variantId: string; variantName: string; visitors: number; conversions: number; conversionRate: number; revenue?: number; averageOrderValue?: number; confidenceInterval: { lower: number; upper: number; }; standardError: number; } export interface StatisticalSummary { testId: string; totalVisitors: number; totalConversions: number; overallConversionRate: number; chiSquare?: number; pValue?: number; degreesOfFreedom?: number; power?: number; } export interface TrendAnalysisInput { tenantId: string; shopId: string; taskId: string; traceId: string; businessType: 'TOC' | 'TOB'; testId: string; granularity: 'HOURLY' | 'DAILY' | 'WEEKLY'; startDate: Date; endDate: Date; } export interface TrendAnalysisResult { success: boolean; testId: string; granularity: string; trends: TrendDataPoint[]; insights: string[]; } export interface TrendDataPoint { timestamp: Date; variantId: string; variantName: string; visitors: number; conversions: number; conversionRate: number; cumulativeVisitors: number; cumulativeConversions: number; cumulativeConversionRate: number; } export interface SegmentAnalysisInput { tenantId: string; shopId: string; taskId: string; traceId: string; businessType: 'TOC' | 'TOB'; testId: string; segmentBy: 'DEVICE' | 'GEO' | 'REFERRER' | 'USER_TYPE' | 'TIME'; } export interface SegmentAnalysisResult { success: boolean; testId: string; segmentBy: string; segments: SegmentData[]; insights: string[]; } export interface SegmentData { segmentName: string; segmentValue: string; variants: VariantSegmentData[]; winner?: { variantId: string; variantName: string; lift: number; }; } export interface VariantSegmentData { variantId: string; variantName: string; visitors: number; conversions: number; conversionRate: number; lift: number; } export class ABTestAnalysisService { static async analyzeABTest(input: ABTestAnalysisInput): Promise { const { tenantId, shopId, taskId, traceId, businessType, testId, analysisType, confidenceLevel = 0.95, minSampleSize = 100 } = input; logger.info(`[ABTestAnalysisService] Analyzing A/B test - testId: ${testId}, type: ${analysisType}, tenantId: ${tenantId}, traceId: ${traceId}`); try { const test = await db('cf_ab_tests') .where({ id: testId, tenant_id: tenantId, shop_id: shopId }) .first(); if (!test) { throw new Error(`A/B test not found: ${testId}`); } const variants = await db('cf_ab_test_variants') .where({ test_id: testId }) .select('*'); if (variants.length < 2) { throw new Error('A/B test must have at least 2 variants'); } const variantAnalysis: VariantAnalysis[] = []; let controlVariant: VariantAnalysis | null = null; for (const variant of variants) { const stats = await this.getVariantStats(testId, variant.id, analysisType); const conversionRate = stats.visitors > 0 ? stats.conversions / stats.visitors : 0; const standardError = Math.sqrt((conversionRate * (1 - conversionRate)) / stats.visitors); const zScore = this.getZScore(confidenceLevel); const analysis: VariantAnalysis = { variantId: variant.id, variantName: variant.name, visitors: stats.visitors, conversions: stats.conversions, conversionRate, revenue: stats.revenue, averageOrderValue: stats.conversions > 0 ? (stats.revenue || 0) / stats.conversions : 0, confidenceInterval: { lower: Math.max(0, conversionRate - zScore * standardError), upper: Math.min(1, conversionRate + zScore * standardError), }, standardError, }; variantAnalysis.push(analysis); if (variant.is_control) { controlVariant = analysis; } } if (!controlVariant && variantAnalysis.length > 0) { controlVariant = variantAnalysis[0]; } let winner: { variantId: string; variantName: string; improvement: number; confidence: number } | undefined; let isStatisticallySignificant = false; if (controlVariant) { const bestVariant = variantAnalysis.reduce((best, current) => current.conversionRate > best.conversionRate ? current : best ); if (bestVariant.variantId !== controlVariant.variantId) { const pooledSE = Math.sqrt( Math.pow(bestVariant.standardError, 2) + Math.pow(controlVariant.standardError, 2) ); const zScore = (bestVariant.conversionRate - controlVariant.conversionRate) / pooledSE; const pValue = this.calculatePValue(zScore); isStatisticallySignificant = pValue < (1 - confidenceLevel); if (isStatisticallySignificant && bestVariant.visitors >= minSampleSize) { winner = { variantId: bestVariant.variantId, variantName: bestVariant.variantName, improvement: ((bestVariant.conversionRate - controlVariant.conversionRate) / controlVariant.conversionRate) * 100, confidence: confidenceLevel * 100, }; } } } const recommendation = this.generateRecommendation(variantAnalysis, winner, isStatisticallySignificant, minSampleSize); await db('cf_ab_test_analyses').insert({ id: `ANALYSIS-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, test_id: testId, tenant_id: tenantId, shop_id: shopId, task_id: taskId, trace_id: traceId, business_type: businessType, analysis_type: analysisType, confidence_level: confidenceLevel, winner_variant_id: winner?.variantId, improvement: winner?.improvement, is_significant: isStatisticallySignificant, recommendation, analyzed_at: new Date(), }); logger.info(`[ABTestAnalysisService] A/B test analysis completed - testId: ${testId}, winner: ${winner?.variantName || 'none'}`); return { success: true, testId, analysisType, analyzedAt: new Date(), variants: variantAnalysis, winner, recommendation, isStatisticallySignificant, }; } catch (error: any) { logger.error(`[ABTestAnalysisService] A/B test analysis failed - testId: ${testId}, error: ${error.message}`); throw error; } } static async analyzeTrends(input: TrendAnalysisInput): Promise { const { tenantId, shopId, taskId, traceId, businessType, testId, granularity, startDate, endDate } = input; logger.info(`[ABTestAnalysisService] Analyzing trends - testId: ${testId}, granularity: ${granularity}, tenantId: ${tenantId}, traceId: ${traceId}`); try { const variants = await db('cf_ab_test_variants') .where({ test_id: testId }) .select('*'); const trends: TrendDataPoint[] = []; const insights: string[] = []; const timeIntervals = this.generateTimeIntervals(startDate, endDate, granularity); for (const interval of timeIntervals) { for (const variant of variants) { const stats = await this.getVariantStatsForPeriod(testId, variant.id, interval.start, interval.end); const cumulativeStats = await this.getVariantCumulativeStats(testId, variant.id, interval.end); trends.push({ timestamp: interval.start, variantId: variant.id, variantName: variant.name, visitors: stats.visitors, conversions: stats.conversions, conversionRate: stats.visitors > 0 ? stats.conversions / stats.visitors : 0, cumulativeVisitors: cumulativeStats.visitors, cumulativeConversions: cumulativeStats.conversions, cumulativeConversionRate: cumulativeStats.visitors > 0 ? cumulativeStats.conversions / cumulativeStats.visitors : 0, }); } } const trendInsights = this.analyzeTrendPatterns(trends); insights.push(...trendInsights); logger.info(`[ABTestAnalysisService] Trend analysis completed - testId: ${testId}, dataPoints: ${trends.length}`); return { success: true, testId, granularity, trends, insights, }; } catch (error: any) { logger.error(`[ABTestAnalysisService] Trend analysis failed - testId: ${testId}, error: ${error.message}`); throw error; } } static async analyzeSegments(input: SegmentAnalysisInput): Promise { const { tenantId, shopId, taskId, traceId, businessType, testId, segmentBy } = input; logger.info(`[ABTestAnalysisService] Analyzing segments - testId: ${testId}, segmentBy: ${segmentBy}, tenantId: ${tenantId}, traceId: ${traceId}`); try { const variants = await db('cf_ab_test_variants') .where({ test_id: testId }) .select('*'); const segments = await this.getSegmentData(testId, variants, segmentBy); const insights = this.generateSegmentInsights(segments); logger.info(`[ABTestAnalysisService] Segment analysis completed - testId: ${testId}, segments: ${segments.length}`); return { success: true, testId, segmentBy, segments, insights, }; } catch (error: any) { logger.error(`[ABTestAnalysisService] Segment analysis failed - testId: ${testId}, error: ${error.message}`); throw error; } } static async getStatisticalSummary(testId: string, tenantId: string): Promise { const test = await db('cf_ab_tests') .where({ id: testId, tenant_id: tenantId }) .first(); if (!test) { throw new Error(`A/B test not found: ${testId}`); } const variants = await db('cf_ab_test_variants') .where({ test_id: testId }) .select('*'); let totalVisitors = 0; let totalConversions = 0; const variantData: { visitors: number; conversions: number }[] = []; for (const variant of variants) { const stats = await this.getVariantStats(testId, variant.id, 'CONVERSION'); totalVisitors += stats.visitors; totalConversions += stats.conversions; variantData.push({ visitors: stats.visitors, conversions: stats.conversions }); } const overallConversionRate = totalVisitors > 0 ? totalConversions / totalVisitors : 0; const chiSquare = this.calculateChiSquare(variantData, totalVisitors, totalConversions); const degreesOfFreedom = variants.length - 1; const pValue = this.calculateChiSquarePValue(chiSquare, degreesOfFreedom); return { testId, totalVisitors, totalConversions, overallConversionRate, chiSquare, pValue, degreesOfFreedom, power: this.calculateStatisticalPower(variantData, overallConversionRate), }; } private static async getVariantStats(testId: string, variantId: string, analysisType: string): Promise<{ visitors: number; conversions: number; revenue: number; }> { const visitors = await db('cf_ab_test_assignments') .where({ test_id: testId, variant_id: variantId }) .count('* as count') .first(); let conversions = 0; let revenue = 0; if (analysisType === 'CONVERSION' || analysisType === 'CTR') { const convResult = await db('cf_ab_test_conversions') .where({ test_id: testId, variant_id: variantId }) .count('* as count') .first(); conversions = parseInt(convResult?.count || '0'); } else if (analysisType === 'REVENUE') { const revResult = await db('cf_ab_test_conversions') .where({ test_id: testId, variant_id: variantId }) .sum('revenue as total') .first(); revenue = parseFloat(revResult?.total || '0'); conversions = await db('cf_ab_test_conversions') .where({ test_id: testId, variant_id: variantId }) .count('* as count') .then(r => parseInt(r[0]?.count || '0')); } return { visitors: parseInt(visitors?.count || '0'), conversions, revenue, }; } private static async getVariantStatsForPeriod( testId: string, variantId: string, startDate: Date, endDate: Date ): Promise<{ visitors: number; conversions: number }> { const visitors = await db('cf_ab_test_assignments') .where({ test_id: testId, variant_id: variantId }) .whereBetween('created_at', [startDate, endDate]) .count('* as count') .first(); const conversions = await db('cf_ab_test_conversions') .where({ test_id: testId, variant_id: variantId }) .whereBetween('created_at', [startDate, endDate]) .count('* as count') .first(); return { visitors: parseInt(visitors?.count || '0'), conversions: parseInt(conversions?.count || '0'), }; } private static async getVariantCumulativeStats( testId: string, variantId: string, endDate: Date ): Promise<{ visitors: number; conversions: number }> { const visitors = await db('cf_ab_test_assignments') .where({ test_id: testId, variant_id: variantId }) .where('created_at', '<=', endDate) .count('* as count') .first(); const conversions = await db('cf_ab_test_conversions') .where({ test_id: testId, variant_id: variantId }) .where('created_at', '<=', endDate) .count('* as count') .first(); return { visitors: parseInt(visitors?.count || '0'), conversions: parseInt(conversions?.count || '0'), }; } private static async getSegmentData( testId: string, variants: any[], segmentBy: string ): Promise { const segments: SegmentData[] = []; const segmentValues = await this.getSegmentValues(testId, segmentBy); for (const segmentValue of segmentValues) { const variantSegments: VariantSegmentData[] = []; for (const variant of variants) { const stats = await this.getSegmentVariantStats(testId, variant.id, segmentBy, segmentValue); const controlStats = await this.getSegmentVariantStats(testId, variants[0].id, segmentBy, segmentValue); const lift = controlStats.conversionRate > 0 ? ((stats.conversionRate - controlStats.conversionRate) / controlStats.conversionRate) * 100 : 0; variantSegments.push({ variantId: variant.id, variantName: variant.name, visitors: stats.visitors, conversions: stats.conversions, conversionRate: stats.conversionRate, lift, }); } const bestVariant = variantSegments.reduce((best, current) => current.conversionRate > best.conversionRate ? current : best ); segments.push({ segmentName: this.getSegmentName(segmentBy), segmentValue, variants: variantSegments, winner: { variantId: bestVariant.variantId, variantName: bestVariant.variantName, lift: bestVariant.lift, }, }); } return segments; } private static async getSegmentValues(testId: string, segmentBy: string): Promise { let query = db('cf_ab_test_assignments') .where({ test_id: testId }) .distinct(segmentBy.toLowerCase()); const results = await query; return results.map((r: any) => r[segmentBy.toLowerCase()]).filter(Boolean); } private static async getSegmentVariantStats( testId: string, variantId: string, segmentBy: string, segmentValue: string ): Promise<{ visitors: number; conversions: number; conversionRate: number }> { const visitors = await db('cf_ab_test_assignments') .where({ test_id: testId, variant_id: variantId }) .where(segmentBy.toLowerCase(), segmentValue) .count('* as count') .first(); const conversions = await db('cf_ab_test_conversions') .where({ test_id: testId, variant_id: variantId }) .where(segmentBy.toLowerCase(), segmentValue) .count('* as count') .first(); const visitorCount = parseInt(visitors?.count || '0'); const conversionCount = parseInt(conversions?.count || '0'); return { visitors: visitorCount, conversions: conversionCount, conversionRate: visitorCount > 0 ? conversionCount / visitorCount : 0, }; } private static getZScore(confidenceLevel: number): number { const zScores: Record = { 0.9: 1.645, 0.95: 1.96, 0.99: 2.576, }; return zScores[confidenceLevel] || 1.96; } private static calculatePValue(zScore: number): number { return 2 * (1 - this.normalCDF(Math.abs(zScore))); } private static normalCDF(x: number): number { const t = 1 / (1 + 0.2316419 * Math.abs(x)); const d = 0.3989423 * Math.exp((-x * x) / 2); const prob = d * t * (0.3193815 + t * (-0.3565638 + t * (1.781478 + t * (-1.821256 + t * 1.330274)))); return x > 0 ? 1 - prob : prob; } private static calculateChiSquare( variantData: { visitors: number; conversions: number }[], totalVisitors: number, totalConversions: number ): number { const expectedRate = totalVisitors > 0 ? totalConversions / totalVisitors : 0; let chiSquare = 0; for (const variant of variantData) { const expectedConversions = variant.visitors * expectedRate; const expectedNonConversions = variant.visitors * (1 - expectedRate); if (expectedConversions > 0) { chiSquare += Math.pow(variant.conversions - expectedConversions, 2) / expectedConversions; } if (expectedNonConversions > 0) { chiSquare += Math.pow((variant.visitors - variant.conversions) - expectedNonConversions, 2) / expectedNonConversions; } } return chiSquare; } private static calculateChiSquarePValue(chiSquare: number, degreesOfFreedom: number): number { return Math.exp(-chiSquare / 2) / Math.sqrt(2 * Math.PI * degreesOfFreedom); } private static calculateStatisticalPower( variantData: { visitors: number; conversions: number }[], baselineRate: number ): number { if (variantData.length < 2) return 0; const effectSize = Math.abs(variantData[1].conversions / variantData[1].visitors - baselineRate); const pooledSE = Math.sqrt((2 * baselineRate * (1 - baselineRate)) / variantData[0].visitors); return effectSize / pooledSE; } private static generateTimeIntervals( startDate: Date, endDate: Date, granularity: 'HOURLY' | 'DAILY' | 'WEEKLY' ): { start: Date; end: Date }[] { const intervals: { start: Date; end: Date }[] = []; let current = new Date(startDate); while (current < endDate) { const intervalStart = new Date(current); let intervalEnd: Date; switch (granularity) { case 'HOURLY': intervalEnd = new Date(current.getTime() + 60 * 60 * 1000); break; case 'DAILY': intervalEnd = new Date(current.getTime() + 24 * 60 * 60 * 1000); break; case 'WEEKLY': intervalEnd = new Date(current.getTime() + 7 * 24 * 60 * 60 * 1000); break; } intervals.push({ start: intervalStart, end: intervalEnd }); current = intervalEnd; } return intervals; } private static analyzeTrendPatterns(trends: TrendDataPoint[]): string[] { const insights: string[] = []; const variantGroups: Record = {}; for (const trend of trends) { if (!variantGroups[trend.variantId]) { variantGroups[trend.variantId] = []; } variantGroups[trend.variantId].push(trend); } for (const [variantId, variantTrends] of Object.entries(variantGroups)) { if (variantTrends.length < 2) continue; const firstHalf = variantTrends.slice(0, Math.floor(variantTrends.length / 2)); const secondHalf = variantTrends.slice(Math.floor(variantTrends.length / 2)); const firstHalfRate = firstHalf.reduce((sum, t) => sum + t.conversionRate, 0) / firstHalf.length; const secondHalfRate = secondHalf.reduce((sum, t) => sum + t.conversionRate, 0) / secondHalf.length; if (secondHalfRate > firstHalfRate * 1.1) { insights.push(`Variant ${variantTrends[0].variantName} shows improving trend (+${(((secondHalfRate - firstHalfRate) / firstHalfRate) * 100).toFixed(1)}%)`); } else if (secondHalfRate < firstHalfRate * 0.9) { insights.push(`Variant ${variantTrends[0].variantName} shows declining trend (${(((secondHalfRate - firstHalfRate) / firstHalfRate) * 100).toFixed(1)}%)`); } } return insights; } private static generateSegmentInsights(segments: SegmentData[]): string[] { const insights: string[] = []; for (const segment of segments) { const bestVariant = segment.variants.reduce((best, current) => current.conversionRate > best.conversionRate ? current : best ); if (bestVariant.lift > 10) { insights.push(`${segment.segmentValue}: ${bestVariant.variantName} performs significantly better (+${bestVariant.lift.toFixed(1)}%)`); } } return insights; } private static getSegmentName(segmentBy: string): string { const names: Record = { DEVICE: 'Device Type', GEO: 'Geography', REFERRER: 'Referrer', USER_TYPE: 'User Type', TIME: 'Time of Day', }; return names[segmentBy] || segmentBy; } private static generateRecommendation( variants: VariantAnalysis[], winner: { variantId: string; variantName: string; improvement: number; confidence: number } | undefined, isSignificant: boolean, minSampleSize: number ): string { if (winner && isSignificant) { return `Winner identified: ${winner.variantName} with ${winner.improvement.toFixed(2)}% improvement at ${winner.confidence}% confidence. Recommend implementing this variant.`; } const underpoweredVariants = variants.filter(v => v.visitors < minSampleSize); if (underpoweredVariants.length > 0) { return `Insufficient sample size for ${underpoweredVariants.length} variant(s). Continue running the test to reach minimum sample size of ${minSampleSize} per variant.`; } if (!isSignificant) { return 'No statistically significant winner found. Consider running the test longer or testing more dramatic changes.'; } return 'Test is inconclusive. Consider reviewing your hypothesis and testing different variations.'; } static async initializeTables(): Promise { if (!(await db.schema.hasTable('cf_ab_test_analyses'))) { await db.schema.createTable('cf_ab_test_analyses', (table) => { table.string('id').primary(); table.string('test_id').notNullable(); table.string('tenant_id').notNullable(); table.string('shop_id').notNullable(); table.string('task_id').notNullable(); table.string('trace_id').notNullable(); table.enum('business_type', ['TOC', 'TOB']).notNullable(); table.enum('analysis_type', ['CONVERSION', 'REVENUE', 'CTR', 'BOUNCE_RATE', 'CUSTOM']).notNullable(); table.decimal('confidence_level', 3, 2).defaultTo(0.95); table.string('winner_variant_id'); table.decimal('improvement', 10, 4); table.boolean('is_significant').defaultTo(false); table.text('recommendation'); table.timestamp('analyzed_at').defaultTo(db.fn.now()); table.foreign('test_id').references('cf_ab_tests.id'); table.index(['test_id']); table.index(['tenant_id', 'shop_id']); table.index(['analyzed_at']); }); logger.info('[ABTestAnalysisService] Created cf_ab_test_analyses table'); } } }