I am running a heterogeneous classification model with numeric, categorical, and unstructured text data to predict a binary response.
The data suffers from class imbalance hence I decided to perform over-sampling to help with this. After fitting the model I was checking the "feature importance" from the random forest and the results are extremely low yet my precision/recall via classification report on the test set are fairly solid (test set does not have over-sampled observations)
# pl
TOKENS_ALPHANUMERIC_HYPHEN = "[A-Za-z0-9\-]+(?=\\s+)"
catTransformer = Pipeline(steps=[
('cat_imputer', SimpleImputer(strategy='constant', fill_value='missing')),
('cat_ohe', OneHotEncoder(handle_unknown='ignore'))])
numTransformer = Pipeline(steps=[
('num_imputer', SimpleImputer(strategy='constant', fill_value=0)),
('num_scaler', StandardScaler())])
textTransformer_0 = Pipeline(steps=[
('text_bow', CountVectorizer(lowercase=True,\
token_pattern=TOKENS_ALPHANUMERIC_HYPHEN,\
stop_words=stopwords))])
textTransformer_1 = Pipeline(steps=[
('text_bow', CountVectorizer(lowercase=True,\
token_pattern=TOKENS_ALPHANUMERIC_HYPHEN,\
stop_words=stopwords))])
FE = ColumnTransformer(
transformers=[
('cat', catTransformer, CAT_FEATURES),
('num', numTransformer, NUM_FEATURES),
('text0', textTransformer_0, TEXT_FEATURES[0]),
('text1', textTransformer_1, TEXT_FEATURES[1])])
PL = Pipeline(steps=[('feature_engineer', FE),
('RF', RandomForestClassifier(n_jobs=-1, class_weight='balanced'))])
RGS = {"RF__max_depth": [100, None],\
"RF__n_estimators": sp_randint(10, 100),\
"RF__max_features": ["auto", "sqrt", "log2", None],\
"RF__bootstrap": [True, False],\
"RF__criterion": ["gini", "entropy"]}
SKF = StratifiedKFold(n_splits=5,\
random_state=11,\
shuffle=True)
cv_model = RandomizedSearchCV(PL, param_distributions=RGS, cv=SKF, n_iter=25)
cv_model.fit(X_train_OS, y_train_OS)
from sklearn.metrics import classification_report, confusion_matrix
preds = cv_model.predict(X_test)
print(confusion_matrix(y_test, preds))
print(classification_report(y_test, preds))
# class report
precision recall f1-score support
CLASS1 0.94 0.99 0.96 2428
CLASS2 0.93 0.67 0.78 495
micro avg 0.94 0.94 0.94 2923
macro avg 0.93 0.83 0.87 2923
weighted avg 0.94 0.94 0.93 2923
# feature importance via sklearn RF classifier
RF_IMPORTANCES = list(zip(cv_model.best_estimator_.named_steps["RF"].feature_importances_, X_train_OS.columns))
RF_IMPORTANCES.sort(reverse=True)
RF_IMPORTANCES
[(0.044093125101590386, 'cat_feature1'),
(0.03352702448927779, 'cat_feature2'),
(0.01581719021567583, 'cat_feature3'),
(0.012946183337756689, 'cat_feature4'),
(0.008118877274266727, 'num_feature1'),
(0.0020503812794265275, 'num_feature2'),
(0.0007034562139435102, 'num_feature3'),
(0.00036099370222021567, 'text_feature1'),
(3.0835853057137074e-05, 'text_feature2')]
Does this mean these feature are completely irrelevant?
If so then how can the accuracy results be fairly decent?
Am I missing something? Thanks!