BibTeX

@article{2506.02881v1,
Author        = {Brian M Cho and Aurélien Bibaut and Nathan Kallus},
Title         = {Simulation-Based Inference for Adaptive Experiments},
Eprint        = {2506.02881v1},
ArchivePrefix = {arXiv},
PrimaryClass  = {stat.ME},
Abstract      = {Multi-arm bandit experimental designs are increasingly being adopted over
standard randomized trials due to their potential to improve outcomes for study
participants, enable faster identification of the best-performing options,
and/or enhance the precision of estimating key parameters. Current approaches
for inference after adaptive sampling either rely on asymptotic normality under
restricted experiment designs or underpowered martingale concentration
inequalities that lead to weak power in practice. To bypass these limitations,
we propose a simulation-based approach for conducting hypothesis tests and
constructing confidence intervals for arm specific means and their differences.
Our simulation-based approach uses positively biased nuisances to generate
additional trajectories of the experiment, which we call \textit{simulation
with optimism}. Using these simulations, we characterize the distribution
potentially non-normal sample mean test statistic to conduct inference. We
provide guarantees for (i) asymptotic type I error control, (ii) convergence of
our confidence intervals, and (iii) asymptotic strong consistency of our
estimator over a wide variety of common bandit designs. Our empirical results
show that our approach achieves the desired coverage while reducing confidence
interval widths by up to 50%, with drastic improvements for arms not targeted
by the design.},
Year          = {2025},
Month         = {Jun},
Url           = {http://arxiv.org/abs/2506.02881v1},
File          = {2506.02881v1.pdf}
}