I am attempting to train a neural network to fit some data with the following architecture:
model = Sequential()
# layer 1
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.5))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(7, input_dim=X_train.shape[1], activation='linear'))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',precision,recall,f1])
from sklearn.utils import class_weight
split_location = int(X_train.shape[0]*0.2)
X_val = X_train.iloc[-split_location:,:]
y_val = y_train[-split_location:]
X_train = X_train.iloc[:-split_location,:]
y_train = y_train[:-split_location]
class_weight = class_weight.compute_class_weight('balanced'
,np.unique(y_train)
,y_train)
hist = model.fit(X_train.values,y_train.values, epochs=50, batch_size=128,
verbose=1, class_weight=class_weight, validation_data=(X_val.values,y_val.values),
callbacks=callbacks_list, shuffle=True)
After training over 50 epochs the following loss/epoch curve is created:
Train on 5595067 samples, validate on 1398766 samples
{'val_loss': [0.16370630699260016, 0.14828306206858941, 0.15146544815120758, 0.15375939467330821, 0.15551779261887769, 0.14860120066405449, 0.15101792141452278, 0.15322067398239911, 0.14989838258384708, 0.15161947356881347, 0.15003373209527424, 0.15379652393121537, 0.14921822092317205, 0.15657513150095473, 0.15066681185591274, 0.15156379228293962, 0.15549042629966836, 0.1547642705268919, 0.14812599346522948, 0.15559401133163162, 0.15182838778070101, 0.15668203393068822, 0.15228943485377922, 0.14875309533438549, 0.15916740564597445, 0.15213683552928584, 0.15506623448426834, 0.15011504060533004, 0.14852479507366009, 0.15893954981803907, 0.16225582986438905, 0.15146525805303634, 0.15033667792805225, 0.16914694318199597, 0.16295670849446725, 0.14960898253949925, 0.15752695067403114, 0.15046651159047286, 0.15538690253535983, 0.1483665518264097, 0.15382219776837991, 0.15089750749229189, 0.15193008905261657, 0.15946591337601629, 0.15603893797274523, 0.15424115727426163, 0.16436825022268611, 0.15270022897166707, 0.15384569235792123, 0.1537987335397171], 'val_acc': [0.95998615922892039, 0.96255485191947765, 0.9625133867995076, 0.96260275128220163, 0.96269068593317253, 0.9627185676517731, 0.96273072122141945, 0.96275860294002003, 0.96285082708616021, 0.9628701298144221, 0.96290015628060732, 0.9628744193095915, 0.96291159493439216, 0.96129802983486878, 0.96176701464004699, 0.96295448988608534, 0.96196719108128159, 0.96077828600352022, 0.96234895615135052, 0.96086264607518346, 0.96104280487229454, 0.96058096922573177, 0.96123154265974442, 0.96254627292913897, 0.96046944235132969, 0.96074611478975036, 0.96051662679819216, 0.96086765048621425, 0.96188354592547998, 0.96026855099423347, 0.96037793312105102, 0.96072252256631918, 0.9611872178763281, 0.95973021935048464, 0.95958938092575885, 0.96135522310379296, 0.96018848041773963, 0.96077399650835094, 0.96052520578853073, 0.96189927407443421, 0.96037578837346638, 0.9607968738159206, 0.96054450851679263, 0.96000689178890541, 0.96036149005623528, 0.96050018373337642, 0.96025282284527935, 0.96084405826278307, 0.96028999847008001, 0.96041725349343632], 'val_precision': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'val_recall': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'val_f1': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'loss': [0.11360416695320169, 0.10546536095823504, 0.10479448631859302, 0.10418981778050561, 0.1039659311605194, 0.103702686021301, 0.10357415694209053, 0.10354264359677245, 0.1034630489532956, 0.10331432841788932, 0.10321070741231013, 0.10320089277934442, 0.10320657373551526, 0.10311705857085159, 0.10292357970790854, 0.10300325969327125, 0.10297844715406301, 0.10295722614892377, 0.10284421649460301, 0.10297831909763921, 0.10270698092951876, 0.10281013161923433, 0.10273270735592134, 0.10257347440138999, 0.10265744293499827, 0.10271797601243517, 0.10261469129023201, 0.10263282053079729, 0.10261252696064917, 0.1026780428078337, 0.10255737576843736, 0.10257961111556378, 0.10256816326291465, 0.10255526256532981, 0.10240226148003012, 0.10245611412796103, 0.10231028322431224, 0.10249142625327236, 0.10232622460226415, 0.10227482178253805, 0.10252379442308171, 0.10236033192308301, 0.1024380711390645, 0.10242730016772771, 0.10227715899869497, 0.10231983227569338, 0.10240002455150068, 0.10235557845757524, 0.10230408325281737, 0.10230684331953707], 'acc': [0.97190543026548659, 0.97220140527360976, 0.97222231655132652, 0.9722621730896317, 0.972251985544067, 0.97227236063468503, 0.97229791886265871, 0.97227003715963645, 0.97221892070268778, 0.97229023352189625, 0.97229255699720052, 0.97226413910656828, 0.972257347409898, 0.97229112716624411, 0.97229023352152344, 0.97227754377241349, 0.97225895596946832, 0.97228058216231172, 0.97230345945812624, 0.97226699876897116, 0.97231632793673428, 0.9722970252188009, 0.97228004597658724, 0.97230828513746581, 0.97228951860630086, 0.97229774013441761, 0.97229416555677084, 0.97231489810626781, 0.97229309318369195, 0.972331341161777, 0.97230685530650929, 0.97233598811238542, 0.97229523793049955, 0.97232008124297364, 0.97233580938363307, 0.97232079615833467, 0.97233580938351583, 0.97231042988452898, 0.97235457591519214, 0.9723451032844771, 0.97233384336595075, 0.97232937514458473, 0.97234331599641011, 0.97232544310908164, 0.97233223480624198, 0.97233223480624198, 0.97231239590172125, 0.97235493337255841, 0.97233312845047248, 0.97234939277734644], 'precision': [0.04318607766310522, 0.082003656892638846, 0.087936882887009502, 0.096868178110347522, 0.099232165040319037, 0.1024872225549192, 0.10610526654919129, 0.10469945686933077, 0.10209030165267655, 0.10681713157497037, 0.10784432216864963, 0.1058086243439857, 0.10735499820198338, 0.1077955172680939, 0.11103456057603048, 0.11026880283592153, 0.10812456906611245, 0.10939158982633816, 0.11280831344917516, 0.1097282671019669, 0.1151799260844769, 0.11271685880418512, 0.11228861765089891, 0.11627422337500112, 0.11250900215032296, 0.11288838400694001, 0.11254026792650559, 0.11347137382598144, 0.11281784569077763, 0.1166189079019115, 0.11469443740454316, 0.11790499302841405, 0.11417370657368192, 0.11577555283141153, 0.11664178536527751, 0.11659183654036762, 0.11873162597643629, 0.11629328767139382, 0.11931309050993898, 0.11795325336546804, 0.11712817784400219, 0.11789546092726166, 0.11829428844281956, 0.11606337103263677, 0.11938934825458292, 0.11785961988148966, 0.11916553174210395, 0.11842773939231339, 0.11874878387905222, 0.12049203373247321], 'recall': [0.014118418249225056, 0.027188699049355843, 0.028828338109625453, 0.032226277240763532, 0.033168871895504236, 0.03481655664606205, 0.035325909798926393, 0.034886021392624637, 0.034452885693979196, 0.035812260258083491, 0.036267397355529793, 0.035399138008983914, 0.035810657334376439, 0.03627785279360974, 0.036985823457266746, 0.036801891997815944, 0.036603202101197038, 0.037376197971835057, 0.038004928222614275, 0.037187494096896555, 0.038622951151551253, 0.038517150244883204, 0.037908066189687749, 0.039557352710400825, 0.03739064596795276, 0.037746901196957429, 0.038091013292955604, 0.038178636542289847, 0.03817338583699121, 0.039149470615628131, 0.039315676229646913, 0.039793657080052496, 0.038753742073928746, 0.038895742302263753, 0.039518730070634445, 0.039408404945255784, 0.040252873469600312, 0.039297097320055686, 0.039975109128980661, 0.040092983292862892, 0.039883563143134244, 0.040358348552354206, 0.039881546274861554, 0.038946843964483412, 0.04038412521416164, 0.040044624047074949, 0.039948125888274684, 0.040334897428796847, 0.039770934152006179, 0.040589943225833217], 'f1': [0.019788689361876124, 0.038075473533327256, 0.040716605544493363, 0.045158392119827175, 0.046441366407507095, 0.048433459572209722, 0.049478664661813394, 0.048941300105780829, 0.048171271840957271, 0.050066211408583493, 0.050794045410133014, 0.049546916480247255, 0.050169278817597321, 0.05070490809166521, 0.051970714730447808, 0.051610791945957142, 0.051066681690763727, 0.05201127497748835, 0.053132730860649895, 0.05193339388246538, 0.054057177250475638, 0.053662245939935828, 0.052853326107522601, 0.055104750658334457, 0.052506291960626908, 0.052858761119297756, 0.053163800431813416, 0.05349706223818005, 0.053174969961761381, 0.054942966279194462, 0.054501893867254576, 0.05565957073787231, 0.054032367220186762, 0.054481634113598784, 0.055028223251572368, 0.055097892515106009, 0.056184053027325626, 0.054852237307409207, 0.056138215439582306, 0.055954590487897728, 0.055656270898052508, 0.056056122042988626, 0.055811011122665961, 0.054504431364966632, 0.056491472101880741, 0.05588381264366022, 0.055920319730667099, 0.056122723342815331, 0.055781426229239844, 0.056669717329863929]}
The orange line is the validation loss, the blue line is the training loss.
The data is highly imbalanced, which is why I use sklearn's class_weight as a parameter for the keras fit. X_train.shape1 = 9, and there is about a 20:1 class 1:class 0 ratio. I would like to make sure that I am interpreting these results correctly. Is the model overfitting after the first epoch? What changes should I investigate to fix this?
--edit-- Sample data, where the last column is the label:
array([[ 2.06469374e-04, 0.00000000e+00, 2.06655044e-04,
1.96226096e-04, 1.96907098e-04, 2.10053075e-04,
2.10123898e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 3.49846068e-04, 0.00000000e+00, 3.14880145e-04,
2.84354598e-04, 2.77412037e-04, 2.72366992e-04,
3.15133305e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 2.75891341e-04, 0.00000000e+00, 2.75906032e-04,
2.83324406e-04, 2.91885604e-04, 2.86806119e-04,
2.91927673e-04, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 2.43793426e-04, 0.00000000e+00, 2.45487050e-04,
2.39993649e-04, 2.31220462e-04, 2.36123428e-04,
2.23352812e-04, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 2.34510576e-04, 0.00000000e+00, 2.36527064e-04,
2.39805622e-04, 2.73673593e-04, 3.27887228e-04,
3.48164995e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 4.20302618e-03, 1.00000000e+00, 4.22147554e-03,
4.05158454e-03, 3.81480614e-03, 3.51370801e-03,
3.75999618e-03, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 1.84195985e-03, 1.00000000e+00, 1.84103369e-03,
1.82596633e-03, 1.78297604e-03, 1.73477764e-03,
1.59173860e-03, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 3.82165605e-04, 0.00000000e+00, 3.82886640e-04,
3.83661466e-04, 3.81056658e-04, 4.42352200e-04,
5.28278411e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 3.20170758e-04, 0.00000000e+00, 3.22507275e-04,
2.78190104e-04, 2.51544925e-04, 2.42750940e-04,
2.87667709e-04, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 4.91506159e-04, 0.00000000e+00, 4.90623484e-04,
5.12634523e-04, 5.22055335e-04, 5.09645568e-04,
5.26666801e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 3.12012480e-04, 0.00000000e+00, 3.09941899e-04,
2.80847154e-04, 2.65126312e-04, 2.36698311e-04,
2.21064672e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 3.61785151e-04, 0.00000000e+00, 3.54394188e-04,
3.45296713e-04, 3.26232058e-04, 2.99172709e-04,
2.57545017e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 1.58464255e-04, 0.00000000e+00, 1.39867938e-04,
1.28589178e-04, 1.12754035e-04, 1.09689470e-04,
1.14503132e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 9.02309913e-05, 0.00000000e+00, 9.02927048e-05,
9.05077389e-05, 9.64206328e-05, 1.11902856e-04,
1.25649507e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 2.29906198e-04, 0.00000000e+00, 2.30416154e-04,
2.04080710e-04, 2.36733009e-04, 2.20505636e-04,
2.12970947e-04, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 1.20649220e-18, 3.41841829e-18,
2.24236368e-18, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 2.79920378e-04, 0.00000000e+00, 2.63137002e-04,
2.29856884e-04, 2.10269492e-04, 1.80452041e-04,
1.48066494e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 7.73335396e-05, 0.00000000e+00, 7.63949598e-05,
7.44459814e-05, 7.25510862e-05, 7.19244684e-05,
1.54948642e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 1.70875910e-04, 0.00000000e+00, 1.97516934e-04,
2.07864758e-04, 2.14027707e-04, 1.83535095e-04,
1.52931429e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 2.88944270e-04, 0.00000000e+00, 2.88389780e-04,
2.87658355e-04, 2.88040997e-04, 2.36616671e-04,
2.20929073e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 9.14578379e-05, 0.00000000e+00, 9.24769902e-05,
9.29960196e-05, 8.17242560e-05, 4.00877913e-05,
2.44654660e-05, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 7.17213115e-04, 0.00000000e+00, 7.17692680e-04,
7.16594468e-04, 7.02347153e-04, 6.46068270e-04,
6.47387334e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 4.20300515e-04, 0.00000000e+00, 4.21534998e-04,
4.26587584e-04, 4.03535119e-04, 3.62465875e-04,
2.80704851e-04, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 5.46763464e-04, 0.00000000e+00, 5.73685876e-04,
5.90085505e-04, 5.64427819e-04, 5.86703901e-04,
5.85212575e-04, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 5.84046724e-03, 1.00000000e+00, 5.10520190e-03,
2.93437589e-03, 1.88552992e-03, 1.27886209e-03,
9.43505709e-04, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 1.08108108e-03, 1.00000000e+00, 1.04567073e-03,
1.01994504e-03, 9.89301968e-04, 9.37864581e-04,
8.85648747e-04, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 1.07275994e-03, 1.00000000e+00, 1.10017538e-03,
1.09289454e-03, 1.10135939e-03, 1.11373804e-03,
1.21099638e-03, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 2.64634584e-02, 1.00000000e+00, 2.64129615e-02,
2.63938540e-02, 2.65509909e-02, 2.67297200e-02,
2.05235897e-02, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 4.64187542e-03, 1.00000000e+00, 4.64035301e-03,
4.42988266e-03, 4.32223732e-03, 3.57887118e-03,
3.35189299e-03, 1.00000000e+00, 0.00000000e+00,
0.00000000e+00],
[ 9.37939659e-04, 0.00000000e+00, 9.40937167e-04,
9.49297837e-04, 9.79787934e-04, 9.77386787e-04,
9.94460058e-04, 0.00000000e+00, 0.00000000e+00,
1.00000000e+00]])