Return to Question

• Comment 1: Yes, X matrix is computed in a very specific way (Exponential Moving averages of the target). It seems that the problem arises particularly well in this case. I'm currently trying to find an example without this "complexity".
• Comment 2: If you are a beginner/intermediate Data Scientist, please refrain for commenting something like "R2 can sometimes be negative": we are in the case of simple OLS with intercept. The Sum of Squares should be minimised, by definition.

• Comment 1: Yes, X matrix is computed in a very specific way (Exponential Moving averages of the target). It seems that the problem arises particularly well in this case. I'm currently trying to find an example without this "complexity".
• Comment 2: If you are a beginner/intermediate Data Scientist, please refrain from commenting something like "R2 can sometimes be negative": we are in the case of simple OLS with intercept. The Sum of Squares should be minimised, by definition.

 import numpy as np; print(np.__version__) # 1.23.5
import scipy; print(scipy.__version__) # 1.10.0
import sklearn as sk; print(sk.__version__) # 1.2.1

from sklearn.linear_model import LinearRegression
import pandas as pd

np.random.seed(8)
s = pd.Series(np.random.normal(10, 1, size=1_000))

l_com = np.arange(100)
df_Xy = pd.concat([s.ewm(com=com).mean() for com in l_com], axis=1)
df_Xy['y'] = s.shift(-1)
df_Xy.dropna(inplace=True)

X = df_Xy[l_com]
y_true = df_Xy.y

model = LinearRegression(fit_intercept=True) # fit_intercept=True by default anyways
model.fit(X, y_true)
print(model.score(X, y_true))
# -0.15802176533843926 = NEGATIVE R2 on VM 1
# -0.05854780689129546 on VM 2 (? dependent on CPU ?)

import numpy as np; print(np.__version__) # 1.23.5
import scipy; print(scipy.__version__) # 1.10.0
import pandas as pd

# Parameters :
(seed, N_obs, N_feat, mu_x, sigma_x, sigma_y) = (0, 100, 1000, 100, 10, 1)

# Building very weird X,y arrays (High Colinearity)
np.random.seed(seed)
s = pd.Series(np.random.normal(mu_x, sigma_x, N_obs))
X_raw = np.ascontiguousarray(np.stack([s.ewm(com=com).mean() for com in np.arange(N_feat)]).T)
y_raw = np.random.normal(0, sigma_y, N_obs)

# Center both arrays to zero
X_offset = X_raw.mean(axis=0)
y_offset = y_raw.mean()
X = X_raw - X_offset
y = y_raw - y_offset

# OLS : Finding parameters that minimise Square Residuals :
p, _,_,_ = scipy.linalg.lstsq(X, y) # <-- This is silently Failing ! (resulting parameters are worst than the zero vector)
pred = np.matmul(X, p)
RSS = np.sum(np.power(y - pred, 2)) # 108.3406316733817
TSS = np.sum(np.power(y - np.mean(y), 2)) # 107.05357955882408

Thanks a lot for your help / suggestions !

import numpy as np; print(np.__version__) # 1.23.5
import scipy; print(scipy.__version__) # 1.10.0
import sklearn as sk; print(sk.__version__) # 1.2.1

from sklearn.linear_model import LinearRegression
import pandas as pd

np.random.seed(8)
s = pd.Series(np.random.normal(10, 1, size=1_000))

l_com = np.arange(100)
df_Xy = pd.concat([s.ewm(com=com).mean() for com in l_com], axis=1)
df_Xy['y'] = s.shift(-1)
df_Xy.dropna(inplace=True)

X = df_Xy[l_com]
y_true = df_Xy.y

model = LinearRegression(fit_intercept=True) # fit_intercept=True by default anyways
model.fit(X, y_true)
print(model.score(X, y_true))
# -0.15802176533843926 = NEGATIVE R2 on VM 1
# -0.05854780689129546 on VM 2 (? dependent on CPU ?)

import numpy as np; print(np.__version__) # 1.23.5
import scipy; print(scipy.__version__) # 1.10.0
import pandas as pd

# Parameters:
(seed, N_obs, N_feat, mu_x, sigma_x, sigma_y) = (0, 100, 1000, 100, 10, 1)

# Building very weird X,y arrays (High Colinearity)
np.random.seed(seed)
s = pd.Series(np.random.normal(mu_x, sigma_x, N_obs))
X_raw = np.ascontiguousarray(np.stack([s.ewm(com=com).mean() for com in np.arange(N_feat)]).T)
y_raw = np.random.normal(0, sigma_y, N_obs)

# Center both arrays to zero
X_offset = X_raw.mean(axis=0)
y_offset = y_raw.mean()
X = X_raw - X_offset
y = y_raw - y_offset

# OLS: Finding parameters that minimise Square Residuals:
p, _,_,_ = scipy.linalg.lstsq(X, y) # <-- This is silently Failing! (resulting parameters are worst than the zero vector)
pred = np.matmul(X, p)
RSS = np.sum(np.power(y - pred, 2)) # 108.3406316733817
TSS = np.sum(np.power(y - np.mean(y), 2)) # 107.05357955882408

import numpy as np; print(np.__version__) # 1.23.5
import scipy; print(scipy.__version__) # 1.10.0
import pandas as pd

np.random.seed(0)
N = 1000
s = pd.Series(np.random.normal(10, 1, size=N))
X = np.ascontiguousarray(np.stack([s.ewm(com=com).mean() for com in np.arange(100)]).T)
y = np.random.normal(10, 1, size=N)

p, _,_,_ = scipy.linalg.lstsq(X, y) # <-- This is silently Failing !
pred = np.matmul(X, p)
RSS = np.sum(np.power(y - pred, 2))
TSS = np.sum(np.power(y - np.mean(y), 2))
assert RSS < TSS # FAIL

import numpy as np; print(np.__version__) # 1.23.5
import scipy; print(scipy.__version__) # 1.10.0
import pandas as pd

# Parameters :
(seed, N_obs, N_feat, mu_x, sigma_x, sigma_y) = (0, 100, 1000, 100, 10, 1)

# Building very weird X,y arrays (High Colinearity)
np.random.seed(seed)
s = pd.Series(np.random.normal(mu_x, sigma_x, N_obs))
X_raw = np.ascontiguousarray(np.stack([s.ewm(com=com).mean() for com in np.arange(N_feat)]).T)
y_raw = np.random.normal(0, sigma_y, N_obs)

# Center both arrays to zero
X_offset = X_raw.mean(axis=0)
y_offset = y_raw.mean()
X = X_raw - X_offset
y = y_raw - y_offset

# OLS : Finding parameters that minimise Square Residuals :
p, _,_,_ = scipy.linalg.lstsq(X, y) # <-- This is silently Failing ! (resulting parameters are worst than the zero vector)
pred = np.matmul(X, p)
RSS = np.sum(np.power(y - pred, 2)) # 108.3406316733817
TSS = np.sum(np.power(y - np.mean(y), 2)) # 107.05357955882408