EDIT
This simulation isn't about ROCAUC, but it does show that there can be a significant difference despite overlapping confidence intervals.
set.seed(2023)
N1 <- 10
N2 <- N1
alpha <- 0.05
R <- 10000
reject_with_ci_overlap <- rep(0, R)
for (i in 1:R){
# Simulate data
#
x1 <- rnorm(N1, 0, 1)
x2 <- rnorm(N2, 0.5, 1)
# Set trackers of conditions being met
#
tracker_overlap <- 0
tracker_psignif <- 0
# Determine confidence intervals for each mean
#
ci1 <- t.test(x1, conf.level = 1 - alpha)$conf.int
ci2 <- t.test(x2, conf.level = 1 - alpha)$conf.int
# Check if the confidence intervals overlap
#
if (
ci1[2] <= ci2[2] # Is the upper limit of ci2 above the upper limit of ci1?
&
ci1[2] >= ci2[1] # Is the upper limit of ci1 above the lower limit of ci2?
){
tracker_overlap <- tracker_overlap + 1 # Then there is overlap
}
if (
ci2[2] <= ci1[2] # Is the upper limit of ci1 above the upper limit of ci2?
&
ci2[2] >= ci1[1] # Is the upper limit of ci2 above the lower limit of ci1?
){
tracker_overlap <- tracker_overlap + 1 # Then there is overlap
}
# Check if the two-sample t-test p-value is below alpha
#
if (t.test(x1, x2, var.equal = F)$p.value <= alpha){
tracker_psignif <- 1
}
# Check if both conditions are met; if so, put a 1 in place i
# of reject_with_ci_overlap
#
if (tracker_overlap > 0 & tracker_psignif > 0){
reject_with_ci_overlap[i] <- 1
}
# Print progress
#
if (i %% 1000 == 0 | i < 5){
print(paste(
round(i/R*100, 2), "% complete",
sep = ""
))
}
}
# Calculate the percentage of iterations where confidence intervals overlap
# yet the two-sample test rejects the null
#
mean(reject_with_ci_overlap) * 100 # 14.19%
I get this behavior in $14.19\%$ of the iterations, so this is far from unusual.