Network in Network in keras implementation I have implemented a keras version of Network in Network Paper for Image Classification for college assignment but when I am running it in my system hangs.
I would like to know whether I have implemented it properly according to architecture, loss function, layers etc as am I new to deep learning and I couldn't find any proper figure stating the number filters in each layer. The code is below.
from keras.datasets import mnist
import numpy as np
from keras.utils.np_utils import to_categorical
from keras.layers import Flatten
from keras.layers import Activation
from keras.layers import Input, Dense, Conv2D, MaxPooling2D,AveragePooling2D,Reshape
from keras.models import Model
from keras import backend as K

def preprocess():
    (x_train,y_train),(x_test,y_test) = mnist.load_data()
    x_train = x_train.astype('float32') / 255.
    x_test = x_test.astype('float32') / 255.
    x_train = np.reshape(x_train, (len(x_train),28,28, 1))
    x_test = np.reshape(x_test, (len(x_test),28,28, 1))
    y_train = to_categorical(y_train,10)
    y_test = to_categorical(y_test,10)

    return (x_train,y_train,x_test,y_test)

def model(x):
    x1 = Conv2D(11,(1,1),padding = 'same')(x)
    x1 = Flatten()(x1)
    x1 = Dense(900,activation = 'relu')(x1)
    x1 = Dense(900,activation = 'relu')(x1)
    x1 = Reshape((30,30,1),input_shape = x1.shape)(x1)
    x1 = Conv2D(11,(1,1),padding = 'same')(x1)
    x1 = Flatten()(x1)
    x1 = Dense(400,activation = 'relu')(x1)
    x1 = Dense(400,activation = 'relu')(x1)
    x1 = Reshape((20,20,1),input_shape = x1.shape)(x1)
    x1 = Conv2D(11,(1,1),padding = 'same')(x1)
    x1 = Flatten()(x1)
    x1 = Dense(100,activation = 'relu')(x1)
    x1 = Dense(100,activation = 'relu')(x1)
    x1 = Reshape((10,10,1),input_shape = x1.shape)(x1)
    x1 = Conv2D(11,(1,1),padding = 'same')(x1)
    x1 = AveragePooling2D((2,2),padding = 'same')(x1)
    x1 = Flatten()(x1)
    x1 = Dense(10)(x1)
    output = Activation('softmax')(x1)
    return output

input_img = Input(shape = (28,28,1))
NiN = Model(input_img,model(input_img))
NiN.compile(optimizer='adadelta',loss = 'mean_absolute_percentage_error')
x_train,y_train,x_test,y_test = preprocess()
NiN.fit(x_train,y_train,
            epochs=10,
            batch_size=128,
            shuffle=True)

 A: I am sorry to say but your implementation is nowhere close to the NiN paper implementation. I just wrote the Keras code which, to my best knowledge, closely approximates the method used in the paper. I get 0.54% Test Error on MNIST which is close enough to their 0.47%. Further hyperparameter tuning would improve my result.
def NiNBlock(kernel, mlps, strides):
    def inner(x):
        l = Conv2D(mlps[0], kernel, strides=strides, padding='same')(x)
        l = Activation('relu')(l)
        for size in mlps[1:]:
            l = Conv2D(size, 1, strides=[1,1])(l)
            l = Activation('relu')(l)
        return l
    return inner


def get_model(img_rows, img_cols):
    img = Input(shape=(img_rows, img_cols, 1))
    l1 = NiNBlock(5, [192, 160, 96], [1,1])(img)
    l1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(l1)
    l1 = Dropout(0.7)(l1)

    l2 = NiNBlock(5, [192, 192, 192], [1,1])(l1)
    l2 = AveragePooling2D(pool_size=(3, 3), strides=(2, 2))(l2)
    l2 = Dropout(0.7)(l2)

    l3 = NiNBlock(3, [192, 192, 10], [1,1])(l2)

    l4 = GlobalAveragePooling2D()(l3)
    l4 = Activation('softmax')(l4)

    model = Model(inputs=img, outputs=l4)
    return model

model = get_model(img_rows, img_cols)
model.summary()

model.compile(loss=keras.losses.categorical_crossentropy,
          optimizer='adam',
          metrics=['accuracy'])

model_name = 'NiN_mnist'
filepath = "{}.weights.best.hdf5".format(model_name)
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, 
save_best_only=True, mode='max')
callbacks_list = [checkpoint]

model.fit(x_train, y_train,
      batch_size=batch_size,
      epochs=epochs,
      verbose=1,
      validation_data=(x_val, y_val),
      callbacks=callbacks_list)

A: I took your implementation and changed the loss function to 'categorical_crossentropy' (used for classification where classes are mutually exclusive) and the optimizer to 'Adam'. Seems to run smooth now. 
A: The above two functions including the consecutive assignment statements that are too complex. I think that the recursion is more elegant and powerful. In addition, it is much easier to be understood.
from tensorflow.keras.layers import Input, Conv2D, MaxPool2D, Dropout, GlobalAveragePooling2D
from tensorflow.keras.models import Model


def nin(input_shape):
    # Define Network in Network model
    input = Input(shape=input_shape)
    x = Conv2D(filters=192, kernel_size=(5,5), activation='relu')(input)
    x = Conv2D(filters=160, kernel_size=(1,1), activation='relu')(x)
    x = Conv2D(filters=96, kernel_size=(1,1), activation='relu')(x)
    x = MaxPool2D(2, strides=2, padding='same')(x)
    x = Dropout(0.5)(x)

    x = Conv2D(filters=192, kernel_size=(5,5), activation='relu')(x)
    x = Conv2D(filters=192, kernel_size=(1,1), activation='relu')(x)
    x = Conv2D(filters=192, kernel_size=(1,1), activation='relu')(x)
    x = MaxPool2D(2, strides=2, padding='same')(x)
    x = Dropout(0.5)(x)

    x = Conv2D(filters=192, kernel_size=(3,3), activation='relu')(x)
    x = Conv2D(filters=192, kernel_size=(1,1), activation='relu')(x)
    x = Conv2D(filters=10, kernel_size=(1,1), activation='relu')(x)

    output = GlobalAveragePooling2D()(x)

    model = Model(input, output)

    return model 


if __name__ == '__main__':
    
    model = nin(input_shape=(28,28,1))

    model.summary()

