I'm trying to make a LSTM model for detecting failures on a physical system, by supplying 27 features of sensor data. I've inputted three disjunct timeseries, each beginning with "normal" operational sensor readings before a failure occurs (each timeseries contains a new type of failure). Each timeseries has 10200 datapoints after splitting the sets in training and testing sets.
My main problem is that the accuracy gets continually degraded and suffers especially during a transition from one timeseries to the next (see attached figure). I'm curious if my implementation has methodical errors, and would be very grateful if someone would point any mistakes.
I've tried changing batch and epoch size, experimenting with different optimizers and layer combinations - but it still shows very strange transitional behaviour during the change from one timeseries to another.
def model_initialize(self, datasets): num_datapoints = 0 for k in datasets: X = datasets[k].iloc[:, :-6] # Prediction variables y = datasets[k].iloc[:, -6:] # num_datapoints = datasets[k].shape + num_datapoints input_dim = 27 # One timestep at a time, ensure that statefulness is enabled. timesteps = 1 batch_size_c = 12 epochs = 100 split_num = 0.7 train_elements = 0 test_elements = 0 if(self.variable_split == 0): deletedRows = 0 split = int(len(datasets[k])*split_num) train_elements = split test_elements = len(datasets[k])-split while True: if ((train_elements%batch_size_c == 0) and (test_elements%batch_size_c == 0)): print("Sets are now divisble by batch size - deleted " + str(deletedRows) +" rows") break datasets[k].drop((datasets[k].shape-1),inplace = True) split = int(len(datasets[k])*split_num) train_elements = split test_elements = len(datasets[k])-split deletedRows = deletedRows + 1 X_train, X_test, y_train, y_test = X[:split], X[split:], y[:split], y[split:] # Feature Scaling scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_test = scaler.transform(X_test) # LSTM expect a 3D matrix, so we reshape the training data with an extra # singular dimension to satisfy the dimension requirements X_train = np.reshape(X_train, (X_train.shape,1,X_train.shape)) X_test = np.reshape(X_test, (X_test.shape,1,X_test.shape)) self.x_testsets[k] = X_test self.y_testsets[k] = y_test if(hasattr(self,'model')): self.model.reset_states() else: self.model = Sequential() self.model.add(LSTM(batch_size_c, return_sequences=True, batch_size = batch_size_c,stateful=True, input_shape=(timesteps, input_dim))) # returns a sequence of vectors of dimension 32 self.model.add(LSTM(batch_size_c, return_sequences=True)) # returns a sequence of vectors of dimension 32 self.model.add(LSTM(batch_size_c)) # return a single vector of dimension 32 self.model.add(Dense(32, activation='softmax')) self.model.add(Dense(32, activation='softmax')) self.model.add(Dense(6, activation='softmax')) opt = SGD(lr=0.003) self.model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) history = np.zeros(epochs) # States are reset at the end of each epoch, and reset at the start of a new timeseries. for i in range(epochs): acc = self.model.fit(X_train, y_train, epochs=1, batch_size=batch_size_c, verbose=1, shuffle=False) history[i] = acc.history['acc'] self.model.reset_states() # Plot accuracy evolution after each timeseries plt.plot(range(0,epochs),history) plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train'], loc='upper left') plt.show() X_test = np.concatenate([self.x_testsets,self.x_testsets,self.x_testsets]) y_test = np.concatenate([self.y_testsets,self.y_testsets,self.y_testsets]) # Ensure that the test dataset is divisible by batch size: deletedRows = 0 while True: if ((len(X_test)%batch_size_c == 0)): print("Test sets now divisible by batch size, deleted " + str(deletedRows) + " rows.") break X_test = np.delete(X_test,(len(X_test)-1),0) deletedRows = deletedRows + 1 y_pred = self.model.predict(X_test,batch_size=batch_size_c) y_pred = (y_pred > 0.5)