How to explain local minima found between two trained Neural networks? I have trained 2 neural networks with SGD and then I have taken a linear path between their weights. Say W_0 and W_1 are the weight matrices of network 1 and network 2, respectively.
Then I compute W_new = (1-t)W_0+tW_1 for t ranging from -1 to 2, with small increments within this range. I then compute the loss for W_new at each t value. I get the following loss (it displays the cross-entropy and the accuracy), shown in the image below .
As you can see, the local minima of N_0 and N_1 happen at t=0 and t=1 as expected. However, there seem to be 2 local minima at t=-0.5 and t=0.5. I find this very strange and it has occurred over many pairs of neural networks.
This is a simple toy network with one hidden layer with 512 neurons and a relu activation.
Any idea as to why this is happening?
 A: I am unable to reproduce your results. Here's my recreation of your graph (cross-entropy only). I used MNIST as a dataset. I suspect (but obviously can't confirm) that you might have a bug in your implementation.

Code:
#!/usr/bin/env python

import itertools as it
from ipdb import set_trace as st
import torch.optim as optim
import torch
import matplotlib.pyplot as plt
import numpy as np
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F

transform = transforms.ToTensor()
trainset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=256, shuffle=True, num_workers=8)
trainloader2 = torch.utils.data.DataLoader(trainset, batch_size=10000, shuffle=False, num_workers=8)
testset = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=10000, shuffle=False, num_workers=8)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(784, 500)
        self.fc2 = nn.Linear(500, 10)        

    def forward(self, x):
        return self.fc2(F.relu(self.fc1(x.view(-1, 784))))

net1 = Net()
net2 = Net()

def interp(net1, net2, t):
    net3 = Net()
    net3.cuda()
    s = 1-t
    net3.fc1.weight = nn.Parameter( s*net1.fc1.weight + t*net2.fc1.weight )
    net3.fc1.bias = nn.Parameter( s*net1.fc1.bias + t*net2.fc1.bias )
    net3.fc2.weight = nn.Parameter( s*net1.fc2.weight + t*net2.fc2.weight )
    net3.fc2.bias = nn.Parameter( s*net1.fc2.bias + t*net2.fc2.bias )
    return net3

def score(net, datas):
    for data in datas: #should be only one batch
        inputs, labels = data
        inputs, labels = inputs.cuda(), labels.cuda()
        out = net(inputs)
        return nn.CrossEntropyLoss()(out, labels).item()

def score_t(net1, net2, t, datas):
    net3 = interp(net1, net2, t)
    return score(net3, datas)

optimizer = optim.Adam(list(net1.parameters()) + list(net2.parameters()), lr=1E-3, weight_decay = 1E-4)

net1.cuda()
net2.cuda()

for epoch in range(50):
    for data in trainloader:
        inputs, labels = data
        inputs, labels = inputs.cuda(), labels.cuda()

        optimizer.zero_grad()
        out1, out2 = net1(inputs), net2(inputs)
        loss = nn.CrossEntropyLoss()(out1, labels) + nn.CrossEntropyLoss()(out2, labels) 
        loss.backward()
        optimizer.step()

    print (epoch, loss.item())

testscores = []
trainscores = []
ts = np.linspace(-1, 2, 91)
for i, t in enumerate(ts):
    print('scoring', i)
    testscores.append(score_t(net1, net2, t, testloader))
    trainscores.append(score_t(net1, net2, t, trainloader2))

plt.plot(ts, trainscores, label='train')
plt.plot(ts, testscores, label='test')
plt.legend()
plt.show()

