Skip to main content
add syntax highlighting
Source Link
import numpy as np

def combine(averages, variances, counts, size=None):
    """
    Combine averages and variances to one single average and variance.

    # Arguments
        averages: List of averages for each part.
        variances: List of variances for each part.
        counts: List of number of elements in each part.
        size: Total number of elements in all of the parts.
    # Returns
        average: Average over all parts.
        variance: Variance over all parts.
    """
    average = np.average(averages, weights=counts)

    # necessary for correct variance in case of multidimensional arrays
    if size is not None:
        counts = counts * size // np.sum(counts, dtype='int')

    squares = (counts - 1) * variances + counts * (averages - average)**2
    return average, np.sum(squares) / (size - 1)
import numpy as np

def combine(averages, variances, counts, size=None):
    """
    Combine averages and variances to one single average and variance.

    # Arguments
        averages: List of averages for each part.
        variances: List of variances for each part.
        counts: List of number of elements in each part.
        size: Total number of elements in all of the parts.
    # Returns
        average: Average over all parts.
        variance: Variance over all parts.
    """
    average = np.average(averages, weights=counts)

    # necessary for correct variance in case of multidimensional arrays
    if size is not None:
        counts = counts * size // np.sum(counts, dtype='int')

    squares = (counts - 1) * variances + counts * (averages - average)**2
    return average, np.sum(squares) / (size - 1)
# sizes k_j and n
ks = np.random.poisson(10, 10)
n = np.sum(ks)

# create data
x = np.random.randn(n, 20)
parts = np.split(x, np.cumsum(ks[:-1]))

# compute statistics on parts
ms = [np.mean(p) for p in parts]
vs = [np.var(p, ddof=1) for p in parts]

# combine and compare
combined = combine(ms, vs, ks, x.size)
numpied = np.mean(x), np.var(x, ddof=1)
distance = np.abs(np.array(combined) - np.array(numpied))
print('combined --- mean:{: .9f} - var:{: .9f}'.format(*combined))
print('numpied  --- mean:{: .9f} - var:{: .9f}'.format(*numpied))
print('distance --- mean:{: .5e} - var:{: .5e}'.format(*distance))
# sizes k_j and n
ks = np.random.poisson(10, 10)
n = np.sum(ks)

# create data
x = np.random.randn(n, 20)
parts = np.split(x, np.cumsum(ks[:-1]))

# compute statistics on parts
ms = [np.mean(p) for p in parts]
vs = [np.var(p, ddof=1) for p in parts]

# combine and compare
combined = combine(ms, vs, ks, x.size)
numpied = np.mean(x), np.var(x, ddof=1)
distance = np.abs(np.array(combined) - np.array(numpied))
print('combined --- mean:{: .9f} - var:{: .9f}'.format(*combined))
print('numpied  --- mean:{: .9f} - var:{: .9f}'.format(*numpied))
print('distance --- mean:{: .5e} - var:{: .5e}'.format(*distance))
import numpy as np

def combine(averages, variances, counts, size=None):
    """
    Combine averages and variances to one single average and variance.

    # Arguments
        averages: List of averages for each part.
        variances: List of variances for each part.
        counts: List of number of elements in each part.
        size: Total number of elements in all of the parts.
    # Returns
        average: Average over all parts.
        variance: Variance over all parts.
    """
    average = np.average(averages, weights=counts)

    # necessary for correct variance in case of multidimensional arrays
    if size is not None:
        counts = counts * size // np.sum(counts, dtype='int')

    squares = (counts - 1) * variances + counts * (averages - average)**2
    return average, np.sum(squares) / (size - 1)
# sizes k_j and n
ks = np.random.poisson(10, 10)
n = np.sum(ks)

# create data
x = np.random.randn(n, 20)
parts = np.split(x, np.cumsum(ks[:-1]))

# compute statistics on parts
ms = [np.mean(p) for p in parts]
vs = [np.var(p, ddof=1) for p in parts]

# combine and compare
combined = combine(ms, vs, ks, x.size)
numpied = np.mean(x), np.var(x, ddof=1)
distance = np.abs(np.array(combined) - np.array(numpied))
print('combined --- mean:{: .9f} - var:{: .9f}'.format(*combined))
print('numpied  --- mean:{: .9f} - var:{: .9f}'.format(*numpied))
print('distance --- mean:{: .5e} - var:{: .5e}'.format(*distance))
import numpy as np

def combine(averages, variances, counts, size=None):
    """
    Combine averages and variances to one single average and variance.

    # Arguments
        averages: List of averages for each part.
        variances: List of variances for each part.
        counts: List of number of elements in each part.
        size: Total number of elements in all of the parts.
    # Returns
        average: Average over all parts.
        variance: Variance over all parts.
    """
    average = np.average(averages, weights=counts)

    # necessary for correct variance in case of multidimensional arrays
    if size is not None:
        counts = counts * size // np.sum(counts, dtype='int')

    squares = (counts - 1) * variances + counts * (averages - average)**2
    return average, np.sum(squares) / (size - 1)
# sizes k_j and n
ks = np.random.poisson(10, 10)
n = np.sum(ks)

# create data
x = np.random.randn(n, 20)
parts = np.split(x, np.cumsum(ks[:-1]))

# compute statistics on parts
ms = [np.mean(p) for p in parts]
vs = [np.var(p, ddof=1) for p in parts]

# combine and compare
combined = combine(ms, vs, ks, x.size)
numpied = np.mean(x), np.var(x, ddof=1)
distance = np.abs(np.array(combined) - np.array(numpied))
print('combined --- mean:{: .9f} - var:{: .9f}'.format(*combined))
print('numpied  --- mean:{: .9f} - var:{: .9f}'.format(*numpied))
print('distance --- mean:{: .5e} - var:{: .5e}'.format(*distance))
Source Link

This is simply an add-on to the answer of aniko with a rough sketch of the derivation and some python code, so all credits go to aniko.

derivation

Let $X_j \in X = \{X_1, X_2, \ldots, X_g\}$ be one of $g$ parts of the data where the number of elements in each part is $k_j = |X_j|$. We define the mean and the variance of each part to be $$\begin{align*} E_j & = \mathrm{E}\left[X_j\right] = \frac{1}{k_j} \sum_{i=1}^{k_j} X_{ji}\\ V_j & = \mathrm{Var}\left[X_j\right] = \frac{1}{k_j-1} \sum_{i=1}^{k_j} (X_{ji} - E_j)^2 \end{align*}$$ respectively. If we set $n = \sum_{j=1}^g k_j$, the variance of the total dataset is given by: $$\begin{align*} \mathrm{Var}\left[X\right] & = \frac{1}{n-1} \sum_{j=1}^{g} \sum_{i=1}^{k_j} (X_{ji} - \mathrm{E}\left[X\right])^2 \\ & = \frac{1}{n-1} \sum_{j=1}^{g} \sum_{i=1}^{k_j} \big((X_{ji} - E_j) - (\mathrm{E}\left[X\right] - E_j)\big)^2 \\ & = \frac{1}{n-1} \sum_{j=1}^{g} \sum_{i=1}^{k_j} (X_{ji} - E_j)^2 - 2(X_{ji} - E_j)(\mathrm{E}\left[X\right] - E_j) + (\mathrm{E}\left[X\right] - E_j)^2 \\ & = \frac{1}{n-1} \sum_{j=1}^{g} (k_j - 1) V_j + k_j (\mathrm{E}\left[X\right] - E_j)^2. \end{align*}$$ If we have the same size $k$ for each part, i.e. $\forall j: k_j = k$, above formula simplifies to $$\begin{align*} \mathrm{Var}\left[X\right] & = \frac{1}{n-1} \sum_{j=1}^g (k-1) V_j + k(g-1) \mathrm{Var}\left[E_j\right] \\ & = \frac{k-1}{n-1} \sum_{j=1}^g V_j + \frac{k(g-1)}{k-1} \mathrm{Var}\left[E_j\right] \end{align*}$$

python code

The following python function works for arrays that have been splitted along the first dimension and implements the "more complex" formula for differently sized parts.

import numpy as np

def combine(averages, variances, counts, size=None):
    """
    Combine averages and variances to one single average and variance.

    # Arguments
        averages: List of averages for each part.
        variances: List of variances for each part.
        counts: List of number of elements in each part.
        size: Total number of elements in all of the parts.
    # Returns
        average: Average over all parts.
        variance: Variance over all parts.
    """
    average = np.average(averages, weights=counts)

    # necessary for correct variance in case of multidimensional arrays
    if size is not None:
        counts = counts * size // np.sum(counts, dtype='int')

    squares = (counts - 1) * variances + counts * (averages - average)**2
    return average, np.sum(squares) / (size - 1)

It can be used as follows:

# sizes k_j and n
ks = np.random.poisson(10, 10)
n = np.sum(ks)

# create data
x = np.random.randn(n, 20)
parts = np.split(x, np.cumsum(ks[:-1]))

# compute statistics on parts
ms = [np.mean(p) for p in parts]
vs = [np.var(p, ddof=1) for p in parts]

# combine and compare
combined = combine(ms, vs, ks, x.size)
numpied = np.mean(x), np.var(x, ddof=1)
distance = np.abs(np.array(combined) - np.array(numpied))
print('combined --- mean:{: .9f} - var:{: .9f}'.format(*combined))
print('numpied  --- mean:{: .9f} - var:{: .9f}'.format(*numpied))
print('distance --- mean:{: .5e} - var:{: .5e}'.format(*distance))