A/B-Testing mit realen Daten

val df = DataFrame.readCsv("data/landing-page-ab-test.csv")
df.head()

// Check for duplicate user_ids
val duplicateUsers = df.groupBy("user_id").count().filter { "count"<Int>() > 1 }
println("Duplicate user_ids: ${duplicateUsers.rowsCount()}")

// Check for mismatched rows (treatment + old_page or control + new_page)
val mismatched = df.filter {
    ("group"<String>() == "treatment" && "landing_page"<String>() == "old_page") ||
    ("group"<String>() == "control" && "landing_page"<String>() == "new_page")
}
println("Mismatched rows: ${mismatched.rowsCount()}")

Duplicate user_ids: 3894
Mismatched rows: 3893

// Remove mismatched rows, then deduplicate by user_id (keep first)
val clean = df
    .filter {
        !("group"<String>() == "treatment" && "landing_page"<String>() == "old_page") &&
        !("group"<String>() == "control" && "landing_page"<String>() == "new_page")
    }
    .distinctBy("user_id")

println("After cleaning: ${clean.rowsCount()} rows (removed ${df.rowsCount() - clean.rowsCount()})")

After cleaning: 290584 rows (removed 3894)

val controlGroup = clean.filter { "group"<String>() == "control" }
val treatmentGroup = clean.filter { "group"<String>() == "treatment" }

val controlConverted = controlGroup.filter { "converted"<Int>() == 1 }.rowsCount()
val controlTotal = controlGroup.rowsCount()
val treatmentConverted = treatmentGroup.filter { "converted"<Int>() == 1 }.rowsCount()
val treatmentTotal = treatmentGroup.rowsCount()

val controlRate = controlConverted.toDouble() / controlTotal
val treatmentRate = treatmentConverted.toDouble() / treatmentTotal

Control:   17489 / 145274 = 0.1204 (12.04%)
Treatment: 17264 / 145310 = 0.1188 (11.88%)
Δ = -0.0016 (-0.16 pp)

val srmTest = proportionZTest(
    successes = controlTotal,
    trials = controlTotal + treatmentTotal,
    p0 = 0.5
)

println("SRM Check (proportion z-test)")
println("  Control:   $controlTotal users")
println("  Treatment: $treatmentTotal users")
println("  Ratio:     ${(controlTotal.toDouble() / (controlTotal + treatmentTotal)).fmt(4)}")
println("  p-value:   ${srmTest.pValue.fmt(4)}")
println("  SRM detected: ${srmTest.isSignificant()}")

SRM Check (proportion z-test)
  Control:   145274 users
  Treatment: 145310 users
  Ratio:     0.4999
  p-value:   0.9468
  SRM detected: false

// Effect size for a 1 pp lift: 12% → 13%
val h = cohensH(p1 = 0.13, p2 = 0.12)
println("Cohen's h for 12% → 13%: ${h.fmt(4)}")

val requiredN = proportionZTestRequiredN(effectSize = h, power = 0.8)
println("Required per group: $requiredN users")
println("Total: ${requiredN * 2} users")

Cohen's h for 12% → 13%: 0.0302
Required per group: 17164 users
Total: 34328 users

// With ~145K per group, what power do we actually have?
val actualPower = proportionZTestPower(effectSize = h, n = 145_000)
println("Power with N=145K per group: ${(actualPower * 100).fmt(1)}%")

val mde = proportionZTestMinimumEffect(n = 145_000, power = 0.8)
println("MDE (Cohen's h): ${mde.fmt(4)}")

Power with N=145K per group: 100.0%
MDE (Cohen's h): 0.0104

val conversionTest = proportionZTest(
    successes1 = treatmentConverted, trials1 = treatmentTotal,
    successes2 = controlConverted,   trials2 = controlTotal
)
val ci = conversionTest.confidenceInterval!!

println("Two-Sample Proportion z-Test")
println("  z-statistic: ${conversionTest.statistic.fmt(4)}")
println("  p-value:     ${conversionTest.pValue.fmt(4)}")
println("  95% CI for Δ(p): [${ci.lower.fmt(4)}, ${ci.upper.fmt(4)}]")
println("  Significant at α=0.05: ${conversionTest.isSignificant()}")

Two-Sample Proportion z-Test
  z-statistic: -1.3109
  p-value:     0.1899
  95% CI for Δ(p): [-0.0039, 0.0008]
  Significant at α=0.05: false

val oneSided = proportionZTest(
    successes1 = treatmentConverted, trials1 = treatmentTotal,
    successes2 = controlConverted,   trials2 = controlTotal,
    alternative = Alternative.GREATER
)
println("One-sided (treatment > control) p = ${oneSided.pValue.fmt(4)}")
println("Significant: ${oneSided.isSignificant()}")

One-sided (treatment > control) p = 0.9051
Significant: false

val effectH = cohensH(p1 = treatmentRate, p2 = controlRate)
println("Cohen's h = ${effectH.fmt(4)}")
println("Interpretation: negligible (|h| < 0.2)")

Cohen's h = -0.0049
Interpretation: negligible (|h| < 0.2)

val controlCI = binomialTest(
    successes = controlConverted,
    trials = controlTotal,
    ciMethod = CIMethod.WILSON
)
val treatmentCI = binomialTest(
    successes = treatmentConverted,
    trials = treatmentTotal,
    ciMethod = CIMethod.WILSON
)

println("Control   Wilson 95% CI: [${controlCI.confidenceInterval!!.lower.fmt(4)}, ${controlCI.confidenceInterval!!.upper.fmt(4)}]")
println("Treatment Wilson 95% CI: [${treatmentCI.confidenceInterval!!.lower.fmt(4)}, ${treatmentCI.confidenceInterval!!.upper.fmt(4)}]")

Control   Wilson 95% CI: [0.1187, 0.1221]
Treatment Wilson 95% CI: [0.1172, 0.1205]

val table = arrayOf(
    intArrayOf(controlConverted, controlTotal - controlConverted),
    intArrayOf(treatmentConverted, treatmentTotal - treatmentConverted)
)
val chiResult = chiSquaredIndependenceTest(table)

println("Chi-squared: χ²=${chiResult.statistic.fmt(4)}, p=${chiResult.pValue.fmt(4)}, df=${chiResult.degreesOfFreedom}")
println("Significant: ${chiResult.isSignificant()}")
println("Consistency check: z²=${(conversionTest.statistic * conversionTest.statistic).fmt(4)}, χ²=${chiResult.statistic.fmt(4)}")

Chi-squared: χ²=1.7185, p=0.1899, df=1.0
Significant: false
Consistency check: z²=1.7185, χ²=1.7185

val posteriorControl = BetaDistribution(
    alpha = 1.0 + controlConverted,
    beta = 1.0 + (controlTotal - controlConverted)
)
val posteriorTreatment = BetaDistribution(
    alpha = 1.0 + treatmentConverted,
    beta = 1.0 + (treatmentTotal - treatmentConverted)
)

println("Posterior Control:   Beta(${1 + controlConverted}, ${1 + controlTotal - controlConverted}), mean=${posteriorControl.mean.fmt(6)}")
println("Posterior Treatment: Beta(${1 + treatmentConverted}, ${1 + treatmentTotal - treatmentConverted}), mean=${posteriorTreatment.mean.fmt(6)}")

Posterior Control:   Beta(17490, 127786), mean=0.120392
Posterior Treatment: Beta(17265, 128047), mean=0.118813

// Monte Carlo: P(treatment > control)
val rng = Random(42)
val nSamples = 100_000
val samplesControl = posteriorControl.sample(nSamples, rng)
val samplesTreatment = posteriorTreatment.sample(nSamples, rng)

val pTreatmentBetter = (0 until nSamples).count {
    samplesTreatment[it] > samplesControl[it]
} / nSamples.toDouble()

println("P(treatment > control) = ${pTreatmentBetter.fmt(4)} (${(pTreatmentBetter * 100).fmt(1)}%)")
println("P(control > treatment) = ${(1.0 - pTreatmentBetter).fmt(4)} (${((1.0 - pTreatmentBetter) * 100).fmt(1)}%)")

P(treatment > control) = 0.0949 (9.5%)
P(control > treatment) = 0.9051 (90.5%)

// 95% Credible intervals
println("Control   95% CI: [${posteriorControl.quantile(0.025).fmt(5)}, ${posteriorControl.quantile(0.975).fmt(5)}]")
println("Treatment 95% CI: [${posteriorTreatment.quantile(0.025).fmt(5)}, ${posteriorTreatment.quantile(0.975).fmt(5)}]")

val diffs = DoubleArray(nSamples) { samplesTreatment[it] - samplesControl[it] }
diffs.sort()
val diffLow = diffs[(nSamples * 0.025).toInt()]
val diffHigh = diffs[(nSamples * 0.975).toInt()]
println("Difference 95% CI: [${diffLow.fmt(5)}, ${diffHigh.fmt(5)}]")
println("Contains zero: ${diffLow <= 0.0 && diffHigh >= 0.0}")

Control   95% CI: [0.11872, 0.12207]
Treatment 95% CI: [0.11715, 0.12048]
Difference 95% CI: [-0.00395, 0.00078]
Contains zero: true

val dfControlRaw = DataFrame.readCsv("data/campaign-control.csv", delimiter = ';')
val dfTestRaw = DataFrame.readCsv("data/campaign-test.csv", delimiter = ';')

// Both CSVs share the same 30 dates. Drop nulls independently, then keep only matched dates.
val dfControlClean = dfControlRaw.dropNulls()
val dfTestClean = dfTestRaw.dropNulls()

val ctrlDates = dfControlClean["Date"].toList().map { it.toString() }.toSet()
val testDates = dfTestClean["Date"].toList().map { it.toString() }.toSet()
val pairedDates = ctrlDates.intersect(testDates)

val dfControlPaired = dfControlClean
    .filter { "Date"<Any>().toString() in pairedDates }
    .sortBy("Date")
val dfTestPaired = dfTestClean
    .filter { "Date"<Any>().toString() in pairedDates }
    .sortBy("Date")

Matched days: 29 (dropped 1 day with missing data)
Control: 29 rows × 10 columns
Test:    29 rows × 10 columns

val controlClicks = dfControlPaired["# of Website Clicks"].toList()
    .map { (it as Number).toDouble() }.toDoubleArray()
val treatmentClicks = dfTestPaired["# of Website Clicks"].toList()
    .map { (it as Number).toDouble() }.toDoubleArray()
val controlImpressions = dfControlPaired["# of Impressions"].toList()
    .map { (it as Number).toDouble() }.toDoubleArray()
val treatmentImpressions = dfTestPaired["# of Impressions"].toList()
    .map { (it as Number).toDouble() }.toDoubleArray()
val controlPurchase = dfControlPaired["# of Purchase"].toList()
    .map { (it as Number).toDouble() }.toDoubleArray()
val treatmentPurchase = dfTestPaired["# of Purchase"].toList()
    .map { (it as Number).toDouble() }.toDoubleArray()

// Rate metrics: normalize by daily impressions
val controlCTR = controlClicks.zip(controlImpressions).map { (c, i) -> c / i }.toDoubleArray()
val treatmentCTR = treatmentClicks.zip(treatmentImpressions).map { (c, i) -> c / i }.toDoubleArray()
val controlPurchaseRate = controlPurchase.zip(controlImpressions).map { (p, i) -> p / i }.toDoubleArray()
val treatmentPurchaseRate = treatmentPurchase.zip(treatmentImpressions).map { (p, i) -> p / i }.toDoubleArray()

Exposure comparison:
  Impressions — Control: 3177233, Treatment: 2123249 (-33.2%)
  Spend       — Control: 66818, Treatment: 74595 (11.6%)

CTR (Control):           mean=5.10%, sd=2.05%
CTR (Treatment):         mean=10.42%, sd=6.82%
Purchase Rate (Control): mean=0.500%, sd=0.217%
Purchase Rate (Treatment): mean=0.848%, sd=0.530%

val ctrDiff = treatmentCTR.zip(controlCTR).map { (t, c) -> t - c }.toDoubleArray()
val ctrDiffNorm = shapiroWilkTest(ctrDiff)

println("CTR differences (treatment - control):")
println("  mean Δ = ${(ctrDiff.describe().mean * 100).fmt(2)} pp")
println("  Shapiro-Wilk: p=${ctrDiffNorm.pValue.fmt(4)}")

CTR differences (treatment - control):
  mean Δ = 5.32 pp
  Shapiro-Wilk: p=0.0012 ✗ non-normal
  Per-group — Control: p=0.2448, Treatment: p=0.0007

val purchRateDiff = treatmentPurchaseRate.zip(controlPurchaseRate)
    .map { (t, c) -> t - c }.toDoubleArray()
val purchRateDiffNorm = shapiroWilkTest(purchRateDiff)

println("Purchase Rate differences (treatment - control):")
println("  mean Δ = ${(purchRateDiff.describe().mean * 100).fmt(3)} pp")
println("  Shapiro-Wilk: p=${purchRateDiffNorm.pValue.fmt(4)}")

Purchase Rate differences (treatment - control):
  mean Δ = 0.348 pp
  Shapiro-Wilk: p=0.0309 ✗ non-normal
  Per-group — Control: p=0.2822, Treatment: p=0.0053

val ctrT = pairedTTest(treatmentCTR, controlCTR)
val ctrW = wilcoxonSignedRankTest(treatmentCTR, controlCTR)

println("CTR — Paired t: t=${ctrT.statistic.fmt(3)}, p=${ctrT.pValue.fmt(4)}")
println("  CI=[${(ctrT.confidenceInterval!!.lower * 100).fmt(2)}%, ${(ctrT.confidenceInterval!!.upper * 100).fmt(2)}%]")
println("  Significant: ${ctrT.isSignificant()}")
println()
println("CTR — Wilcoxon signed-rank: W=${ctrW.statistic.fmt(1)}, p=${ctrW.pValue.fmt(4)}")
println("  Significant: ${ctrW.isSignificant()}")

CTR — Paired t: t=4.016, p=0.0004
  CI=[2.61%, 8.04%]
  Significant: true

CTR — Wilcoxon signed-rank: W=388.0, p=0.0002
  Significant: true

val purchRateT = pairedTTest(treatmentPurchaseRate, controlPurchaseRate)
val purchRateW = wilcoxonSignedRankTest(treatmentPurchaseRate, controlPurchaseRate)

println("Purchase Rate — Paired t: t=${purchRateT.statistic.fmt(3)}, p=${purchRateT.pValue.fmt(4)}")
println("  CI=[${(purchRateT.confidenceInterval!!.lower * 100).fmt(3)}%, ${(purchRateT.confidenceInterval!!.upper * 100).fmt(3)}%]")
println("  Significant: ${purchRateT.isSignificant()}")
println()
println("Purchase Rate — Wilcoxon signed-rank: W=${purchRateW.statistic.fmt(1)}, p=${purchRateW.pValue.fmt(4)}")
println("  Significant: ${purchRateW.isSignificant()}")

Purchase Rate — Paired t: t=3.167, p=0.0037
  CI=[0.123%, 0.574%]
  Significant: true

Purchase Rate — Wilcoxon signed-rank: W=358.0, p=0.0025
  Significant: true

val ctrDiffs = DoubleArray(treatmentCTR.size) { treatmentCTR[it] - controlCTR[it] }
val purchDiffs = DoubleArray(treatmentPurchaseRate.size) {
    treatmentPurchaseRate[it] - controlPurchaseRate[it]
}

val ctrDz = ctrDiffs.mean() / sqrt(ctrDiffs.variance())
val purchDz = purchDiffs.mean() / sqrt(purchDiffs.variance())

println("CTR           — paired dz = ${ctrDz.fmt(3)} (${interpretD(ctrDz)})")
println("Purchase Rate — paired dz = ${purchDz.fmt(3)} (${interpretD(purchDz)})")

CTR           — paired dz = 0.746 (medium)
Purchase Rate — paired dz = 0.588 (medium)

val rawPValues = doubleArrayOf(ctrT.pValue, purchRateT.pValue)

val bonf = bonferroniCorrection(rawPValues)
val holm = holmBonferroniCorrection(rawPValues)
val bh   = benjaminiHochbergCorrection(rawPValues)

Metric      Raw p       Bonferroni    Holm          BH
------------------------------------------------------------
CTR         0.0004      0.0008        0.0008        0.0008
Purch.Rate  0.0037      0.0074        0.0037        0.0037

val allCTR = controlCTR + treatmentCTR
val allPurchaseRate = controlPurchaseRate + treatmentPurchaseRate

val corrP = pearsonCorrelation(allCTR, allPurchaseRate)
val corrS = spearmanCorrelation(allCTR, allPurchaseRate)

println("Pearson  r = ${corrP.coefficient.fmt(3)}, p = ${"%.2e".format(corrP.pValue)}")
println("Spearman ρ = ${corrS.coefficient.fmt(3)}, p = ${"%.2e".format(corrS.pValue)}")

Pearson  r = 0.748, p = 1.55e-11
Spearman ρ = 0.517, p = 3.22e-05