A relatively low number of iid random vectors of a relatively high dimension (10,000) are added up together element wise: $$\sum_{i=1}^{n}X_i=Y$$ where $dim(X_i)=dim(X_j)=dim(Y),\forall i,j$ and $dim(X_i)\gg n$
My modeling suggests that $$E \left[\sum_{i=1}^{n}\rho_{X_i,Y}^2\right]<1$$ for the described case.
More generally $$E \left[\sum_{i=1}^{n}\rho_{X_i,Y}^2\right]<E \left[\sum_{i=1}^{n}r_{X_i,Y}^2\right]$$ and $$E \left[\sum_{i=1}^{n}r_{X_i,Y}^2\right]\ge1$$ where $\rho$ is Spearman's rank correlation coefficient, $r$ is Pearson correlation coefficient.
Computational assumptions
For computing $\rho_{X,Y}^2$ and $r_{X,Y}^2$ in R I use
cor(X, Y, method='spearman')^2
and
cor(X, Y, method='pearson')^2
respectively.
I also assume $X_i \sim \mathcal{N}(0, 1)$ but the following works for some other common distributions as well.
Fixed number of variables, fixed dimension, varied correlation function
Given just 3 of 10,000-vectors the distributions of Spearman's $$\rho_{X_1,Y}^2+\rho_{X_2,Y}^2+\rho_{X_3,Y}^2$$ and Pearson's $$r_{X_1,Y}^2+r_{X_2,Y}^2+r_{X_3,Y}^2$$ are as follows:
Fixed number of variables, varied dimension
Here's how the distributions of $\rho_{X_1,Y}^2+\rho_{X_2,Y}^2+\rho_{X_3,Y}^2$ and $r_{X_1,Y}^2+r_{X_2,Y}^2+r_{X_3,Y}^2$ look like next to each other for various dimensions. It appears as with the increase of the number of dimensions both distributions travel to the left but Pearson's stops at 1.0 while Spearman's continues to move below 1.0.
Fixed dimension, varied number of variables
When the number of variables increases, the Spearman's distribution seems to drift even further below 1.0 in the beginning and then comes back and exceeds 1.0 whereas Pearson's doesn't go below 1.0 at all and bounces back at the high number of variables just as Spearman's does:
Varied dimension, varied number of variables
A 3D plot to corroborate the above.
Questions
- Is my modeling correct?
- Why is the Pearson sum of squares always greater than Spearman's on average?
- Why does the Pearson sum of squares never get below 1.0 on average whereas Spearman's does?
Here's my code:
library(ggplot2)
library(dplyr)
library(purrr)
library(plotly)
r_squared_sum <- function(num_vars, distrib_func, rsq_func, iterations=1000) {
R_sq_sums <- c()
for (i in 1:iterations) {
data <- c()
for (j in 1:num_vars) {
data <- append(data, list(distrib_func()))
}
Y = Reduce(`+`, data)
R_sq_sum = Reduce(`+`, lapply(data, partial(rsq_func, y=Y)))
R_sq_sums <- append(R_sq_sums, R_sq_sum)
}
return(data.frame(r_squared_sum=R_sq_sums))
}
# ---------------------------------------------
# Fixed number of variables, fixed dimensions,
# varied correlation function, histogram
# ---------------------------------------------
data_spearman <- r_squared_sum(
num_vars=3,
distrib_func=partial(rnorm, 10000, 0, 1),
rsq_func=function (x, y) cor(x, y, method='spearman')^2
)
data_spearman$method <- 'spearman'
data_pearson <- r_squared_sum(
num_vars=3,
distrib_func=partial(rnorm, 10000, 0, 1),
rsq_func=function (x, y) cor(x, y, method='pearson')^2
)
data_pearson$method <- 'pearson'
ggplot(
bind_rows(data_pearson, data_spearman),
aes(x=r_squared_sum, fill=method, color=method)
) + geom_histogram(position="identity", alpha=0.8)
# -------------------------------------------------------
# Fixed number of variables, varied dimension, boxplots
# -------------------------------------------------------
dimension_range <- c(10, 100, 1000, 10000)
distrib_func_range <- lapply(
dimension_range,
partial,
.f=partial(rnorm, mean=0, sd=1)
)
data_frames <- lapply(
distrib_func_range,
partial(
r_squared_sum,
num_vars=3,
rsq_func=function (x, y) cor(x, y, method='spearman')^2
)
)
for (i in 1:length(dimension_range)) {
data_frames[[i]]$dimension <- as.character(dimension_range[i])
data_frames[[i]]$method <- 'spearman'
}
data_spearman <- bind_rows(data_frames)
data_frames <- lapply(
distrib_func_range,
partial(
r_squared_sum,
num_vars=3,
rsq_func=function (x, y) cor(x, y, method='pearson')^2
)
)
for (i in 1:length(dimension_range)) {
data_frames[[i]]$dimension <- as.character(dimension_range[i])
data_frames[[i]]$method <- 'pearson'
}
data_pearson <- bind_rows(data_frames)
ggplot(
bind_rows(data_pearson, data_spearman),
aes(y=dimension, x=r_squared_sum, fill=method, color=method)
) + geom_boxplot()
# -------------------------------------------------------
# Fixed dimension, varied number of variables, boxplots
# -------------------------------------------------------
num_vars_range = c(10, 100, 1000)
data_frames <- lapply(
num_vars_range,
partial(
r_squared_sum,
distrib_func=partial(rnorm, 10000, 0, 1),
rsq_func=function (x, y) cor(x, y, method='spearman')^2
)
)
for (i in 1:length(num_vars_range)) {
data_frames[[i]]$num_vars = as.character(num_vars_range[i])
data_frames[[i]]$method <- 'spearman'
}
data_spearman <- bind_rows(data_frames)
data_frames <- lapply(
num_vars_range,
partial(
r_squared_sum,
distrib_func=partial(rnorm, 10000, 0, 1),
rsq_func=function (x, y) cor(x, y, method='pearson')^2
)
)
for (i in 1:length(num_vars_range)) {
data_frames[[i]]$num_vars = as.character(num_vars_range[i])
data_frames[[i]]$method <- 'pearson'
}
data_pearson <- bind_rows(data_frames)
ggplot(
bind_rows(data_pearson, data_spearman),
aes(y=num_vars, x=r_squared_sum, fill=method, color=method)
) + geom_boxplot()
# ---------------------------------------------------------
# Varied dimension, varied number of variables, 3D surfaces
# ---------------------------------------------------------
num_vars_range = seq(10, 100, by=10)
dimension_range <- seq(100, 1000, by=100)
distrib_func_range <- lapply(
dimension_range,
partial,
.f=partial(rnorm, mean=0, sd=1)
)
nrow <- length(num_vars_range)
ncol <- length(distrib_func_range)
data_pearson <- matrix(data=NA, nrow=nrow, ncol=ncol)
for (i in 1:nrow) {
for (j in 1:ncol) {
r_squared_sums <- r_squared_sum(
num_vars_range[[i]],
distrib_func_range[[j]],
function (x, y) cor(x, y, method='pearson')^2,
iterations=100
)
data_pearson[i, j] <- mean(r_squared_sums$r_squared_sum)
}
}
data_spearman <- matrix(data=NA, nrow=nrow, ncol=ncol)
for (i in 1:nrow) {
for (j in 1:ncol) {
r_squared_sums <- r_squared_sum(
num_vars_range[[i]],
distrib_func_range[[j]],
function (x, y) cor(x, y, method='spearman')^2,
iterations=100
)
data_spearman[i, j] <- mean(r_squared_sums$r_squared_sum)
}
}
fig <- plot_ly(
x=dimension_range,
y=num_vars_range,
showscale=FALSE,
showlegend=TRUE,
name='sums of r^2',
width=700,
height=500
) %>% layout(
scene=list(
xaxis=list(title="vectors"),
yaxis=list(title="dimensions"),
zaxis=list(title="mean sum r_sq")
)
)
fig <- fig %>% add_surface(
z=data_spearman,
name='spearman',
colorscale=list(c(0, 1), c("#00BFC4","#00BFC4")),
hovertemplate=paste(' vectors: %{x}<br>', 'dimensions: %{y}<br>', 'mean sum r_sq: %{z}')
)
fig <- fig %>% add_surface(
z=data_pearson,
name='pearson',
colorscale=list(c(0, 1), c("#F8776D","#F8776D")),
hovertemplate=paste(' vectors: %{x}<br>', 'dimensions: %{y}<br>', 'mean sum r_sq: %{z}')
)
fig <- fig %>% add_surface(
z=matrix(data=1, nrow=nrow, ncol=ncol),
name='level',
colorscale=list(c(0, 1), c("#696969","#696969")),
hovertemplate=paste(' vectors: %{x}<br>', 'dimensions: %{y}<br>', 'mean sum r_sq: %{z}')
)
fig
I've implemented similar modeling in Python (my main language) with the same results.