I have a set of data and I am using this code on it:
categorical_columns = ["x3"]
numerical_columns = ["x1", "x2", "x4", "x5", "x6", "x8"]
preprocessor = make_column_transformer(
(OneHotEncoder(drop="first"), categorical_columns),
(StandardScaler(), numerical_columns),
remainder="passthrough",
verbose_feature_names_out=False,
)
model = make_pipeline(
preprocessor,
TransformedTargetRegressor(
regressor=LinearRegression(), **func=np.log10, inverse_func=sp.special.exp10**
),
) # this is because y is positively skewed (1.2 of skewness) and can't have negative values
When I do this and run a linear regression, I get the following metrics:
- R-squared on training set: -1936.24
- R-squared on testing set: -2227.20
- MedAE on training set: 0.80
- MedAE on testing set: 0.81
- MSE on training set: 20747.23
- MSE on test set: 20747.23
If I remove the log transformation, I get the following metrics:
- R-squared on training set: 0.86
- R-squared on testing set: 0.84
- MedAE on training set: 0.54
- MedAE on testing set: 0.53
- MSE on training set: 1.53
- MSE on test set: 1.53
For reference this is the code:
model = make_pipeline(
preprocessor,
TransformedTargetRegressor(
regressor=LinearRegression()
),
)
The thing is then my linear regression starts predicting negative values when $y$ doesn't have a single negative value, yet it does have a lot of values very close to $0$. Using a standard scaler yields the same results has the no log transformation. What am I doing wrong ?
Information that may or may not be relevant - doing a ridge or a lasso regression gives the same results as a linear regression (so probably something is very wrong, right?)
MWE
import numpy as np
import pandas as pd
from sklearn.compose import make_column_transformer
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import make_pipeline
from sklearn.metrics import mean_squared_error, median_absolute_error, r2_score
from sklearn.model_selection import train_test_split
from sklearn.compose import TransformedTargetRegressor
import scipy as sp
X = data.drop(columns = "y")
y = data.y
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
categorical_columns = ["x3"]
numerical_columns = ["x1", "x2", "x4", "x5", "x6", "x8"]
# Preprocessor
preprocessor = make_column_transformer(
(OneHotEncoder(drop="first"), categorical_columns),
(StandardScaler(), numerical_columns),
remainder="passthrough",
verbose_feature_names_out=False,
)
# Model with Log Transformation
model_log = make_pipeline(
preprocessor,
TransformedTargetRegressor(
regressor=LinearRegression(), func=np.log10, inverse_func=sp.special.exp10
),
)
# Model without Log Transformation
model_no_log = make_pipeline(
preprocessor,
LinearRegression(),
)
# Train and evaluate models
for model, name in zip([model_log, model_no_log], ['With Log', 'Without Log']):
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
r2 = r2_score(y_test, y_pred)
medae = median_absolute_error(y_test, y_pred)
mse = mean_squared_error(y_test, y_pred)
print(f"{name}: R-squared: {r2:.2f}, MedAE: {medae:.2f}, MSE: {mse:.2f}")
data sample :
x1, x2, x3,x4, x5, x6, x8, y
67.55,64.07,4,3.71,0.027,0.019,0.022,3.99
30.51,29.74,3,1.47,0.004,0.018,-0.004,0.96
86.74,86.70,3,0.68,0.021,0.018,0.029,4.24
0.97,0.90,1,0.91,0.037,0.017,0.034,0.08
76.41,70.19,4,0.25,0.018,0.020,0.008,10.69
38.59,35.01,3,1.23,0.029,0.017,0.016,5.36
51.75,51.46,3,2.29,0.021,0.016,0.026,1.08
16.87,15.42,3,1.28,0.037,0.016,0.033,2.42
47.12,42.60,3,2.43,0.035,0.016,0.033,5.35
41.20,39.13,2,1.08,0.037,0.018,0.046,3.40
7.35,7.18,2,1.77,0.029,0.017,0.032,0.22
28.23,27.87,4,2.45,0.033,0.021,0.018,1.77
54.31,49.06,1,0.85,0.036,0.015,0.040,5.53
2.46,2.25,4,1.02,0.009,0.017,0.013,0.27
65.19,61.14,1,0.95,0.047,0.015,0.051,4.19
27.18,24.74,2,1.90,0.018,0.019,0.009,2.48
75.26,75.22,3,0.42,0.003,0.014,0.005,0.99
72.62,72.01,2,0.13,0.021,0.016,0.024,3.43
16.22,15.92,1,0.99,0.050,0.022,0.048,0.30
79.54,75.36,3,0.29,0.007,0.019,0.022,5.67
