How to make simple neural network to separate lines? Recently I've read an interesting article about neural networks and what they are doing with data. As a newbie I felt that I had to do the same the author did... And stuck at the very beginning.
First of all I generated two slightly overlapped sine waves and tried to classify points. As the article says after adding hidden layer to nn separation line looks like this:
 
I've tried to achieve the same result but no one combination of layers and neurons in them hasn't give me the same result: separation line is almost a straight line and training accuracy is no more than 0,82. 
Therefore I decided to make a 2d input passing into a model x and y coordinates. I thought that it will give very high training accuracy(something about 0.98-0.99) because every (x,y) point belongs only to one curve. But again I coudn't find nn parameters which would give me higher accuracy than 0.87 and in most cases separation line is almost straight.
That's how my separation looks like. Sorry for 3d plot, I was just playing with matplolib:

Here is my code(most part of it is all about plotting):
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.utils import np_utils
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D


NB_EPOCH = 50


def prepare_data(n_points=256):
    x = np.linspace(0, 3*np.pi, n_points)
    shift1 = 0.8
    shift2 = -0.8

    y1 = np.cos(x*1.2)+shift1
    y2 = np.cos(x*1.2)+shift2

    features1 = zip(x, y1)
    features2 = zip(x, y2)

    labels1 = np.zeros((n_points,))
    labels2 = np.ones((n_points,))

    return np.concatenate([features1, features2]), np.concatenate([labels1, labels2])


def create_model():

    model = Sequential()
    model.add(Dense(4, input_shape=(2,)))
    model.add(Activation('tanh'))
    model.add(Dense(4))
    model.add(Activation('tanh'))
    model.add(Dense(4))
    model.add(Activation('tanh'))
    model.add(Dense(2))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='sgd',
                  metrics=['accuracy'])

    return model


if __name__ == "__main__":
    features, labels = prepare_data()
    f_len = features.shape[0]

    labels = np_utils.to_categorical(labels, nb_classes=2)
    model = create_model()
    print(model.summary())

    model.fit(features, labels, nb_epoch=NB_EPOCH)

    reshaped_features = features.reshape(features.shape[0], 2)
    # ######################################### #
    # Plotting                                  #
    # ######################################### #
    fig = plt.figure(1)

    # ######################################### #
    # Source data                               #
    # ######################################### #
    ax = Axes3D(fig) #fig.add_subplot(311, projection='3d')
    ax.plot(features[:f_len/2,0], np.zeros(f_len/2), features[:f_len/2, 1])
    ax.plot(features[f_len/2:,0], np.zeros(f_len/2), features[f_len/2:, 1])

    # ###########################################
    # Separation curve
    # ###########################################
    x,y = np.meshgrid(np.linspace(0, 3*np.pi, 50), np.linspace(-1.5, 1.5, 36))
    input = []

    for _x, _y in zip(x, y):
        for i in range(len(_x)):
            input.append((_x[i], _y[i]))

    input = np.asarray(input)
    classes = model.predict(input)

    first_class_x = []
    first_class_y = []
    second_class_x = []
    second_class_y = []

    for i in range(len(classes)):
        if classes[i][0] > 0.5:
            first_class_x.append(input[i][0])
            first_class_y.append(input[i][1])
        else:
            second_class_x.append(input[i][0])
            second_class_y.append(input[i][1])

    ax.scatter(first_class_x, np.zeros(len(first_class_x)),
               first_class_y, c='r')

    ax.scatter(second_class_x, np.zeros(len(second_class_x)),
               second_class_y, c='y')

    plt.show()

So, the questions are:


*

*What I'm doing wrong that I can't get high accuracy?

*And how can I plot data transformations as they are shown in article? I also tried to do that but unsuccessfully.

 A: There were some issues with your code.


*

*In Keras, it seems that model.predict_classes() should be used to get the actual classes (not the output probabilities).

*Your training data was not shuffled.


In the blog article that you mentioned, it isn't clear what is the training data. Perhaps it doesn't consist of the actual $((x, y), label)$ for the points on the curves. I suggest you to contact the author and kindly ask him to be more specific about the training data.
I prepared a Python & Keras sample to get you started. Unfortunately, I could only answer (somehow) your second question, the one related to plotting. Instead of your training data I generated random "blobs" of samples.
import numpy as np
from keras.models import Sequential
from keras.layers.core import Dense
from keras.optimizers import SGD
import sklearn.datasets
from matplotlib.pyplot import show, cm, scatter, contourf
from keras.utils import np_utils


def plot2dSamples(X, y):
    distinctClasses = np.unique(y)
    colors = cm.rainbow(np.linspace(0.0, 1.0, distinctClasses.size))

    for currentClass, color in zip(distinctClasses, colors):
        samplesOfClass = X[y == currentClass]
        scatter(samplesOfClass[:, 0], samplesOfClass[:, 1], c=color, s=40)


def drawDecisionSurface(model, xmin, xmax, ymin, ymax, resolution):
    # Create test set (surface)
    x0, x1 = np.meshgrid(np.arange(xmin, xmax, resolution), np.arange(ymin, ymax, resolution))
    X = np.c_[x0.ravel(), x1.ravel()]
    decisions = model.predict_classes(X, verbose=0)
    # Plot
    contourf(x0, x1, decisions.reshape(x0.shape), alpha=0.8)


if __name__ == '__main__':
    np.random.seed(42)
    M = 1000  # number of training samples
    d = 2
    h = 50
    noClasses = 3

    X, y = sklearn.datasets.make_blobs(n_samples=M, n_features=d, centers=noClasses) 
    yhot = np_utils.to_categorical(y, noClasses)   

    model = Sequential()
    model.add(Dense(h, init='uniform', activation="tanh", input_dim=d))
    model.add(Dense(noClasses, init='uniform', activation="softmax"))

    sgd = SGD(lr=0.05)
    model.compile(loss='categorical_crossentropy', optimizer=sgd)
    model.fit(X, yhot, nb_epoch=1, batch_size=64, verbose=0)

    drawDecisionSurface(model, -15, 15, -15, 15, 0.1)
    plot2dSamples(X, y)
    show()


