0
$\begingroup$

I'm new to extreme learning machine (ELM) which is a single layer feedforward neural network. I'm trying to write a ridge version of the classical ELM. But this Ridge-ELM confuses me. I think the coefficient estimates of ridge regression fitted for a linear model should give the same predicted values as ELM-Ridge. But the computations below show that the ELM-Ridge predictions are the same as those of the ELM. I have used the linear activation function in the computations. I have take the number of hidden neurons as 1000 and the ridge parameter lambda as 10. (The code is adapted from here)

elm <- function(X, y, n_hidden=NULL, active_fun=function(x)x) {
  # X: an N observations x p features matrix
  # y: the target
  # n_hidden: the number of hidden nodes
  # active_fun: activation function
  pp1 = ncol(X) + 1
  w0 = matrix(rnorm(pp1*n_hidden), pp1, n_hidden) # random weights
  h = active_fun(cbind(1, X) %*% w0)              # compute hidden layer
  B = MASS::ginv(h) %*% y                         # find weights for hidden layer
  fit = h %*% B                                   # fitted values
  list(fit= fit, loss=crossprod(fit - y), B=B, w0=w0)
}


elm_ridge <- function(X, y, n_hidden=NULL, lambda=0, active_fun=function(x)x) {
  # X: an N observations x p features matrix
  # y: the target
  # n_hidden: the number of hidden nodes
  # active_fun: activation function
  pp1 = ncol(X) + 1
  w0 = matrix(rnorm(pp1*n_hidden), pp1, n_hidden)                      # random weights
  h = active_fun(cbind(1, X) %*% w0)                                   # compute hidden layer
  B = solve(crossprod(h) + lambda*diag(1,ncol(h))) %*% crossprod(h,y)  # find weights for hidden layer
  fit = h %*% B                                                        # fitted values
  list(fit= fit, loss=crossprod(fit - y), B=B, w0=w0)
}

ridge <- function(X, y, lambda){ # A simple ridge function for linear model
  Xc <- scale(X, scale=F)
  yc <- y-mean(y)
  btr <- solve(crossprod(Xc) + lambda*diag(1,ncol(Xc)))%*%crossprod(Xc,yc)
  inter <- mean(y) - sum(colMeans(X)*drop(btr))
  btr <- c(inter,btr)
  btr
}




set.seed(1)
# training
x <- matrix(rnorm(1000),100)
y <- matrix(rnorm(100),100)

# test
xt <- matrix(rnorm(200),20)
yt <- matrix(rnorm(20),20)


# Compute predicted values on test set

# ELM
bt.elm <- elm(x,y,1000)
pred.elm <- cbind(1,xt) %*% bt.elm$w0 %*% bt.elm$B

# ELM ridge, lambda=10
bt.elm.r1 <- elm_ridge(x,y,1000,10)
pred.elm.r1 <- cbind(1,xt) %*% bt.elm.r1$w0 %*% bt.elm.r1$B


# Ridge, lambda=10
bt.r1 <- ridge(x,y,10)
pred.r1 <- cbind(1,xt) %*% bt.r1

Output:

data.frame(ELM=pred.elm,ELM.ridge=pred.elm.r1,Ridge=pred.r1)


#     ELM         ELM.ridge   Ridge
# 1   0.05653623  0.05652927  0.048668547
# 2   0.50931986  0.50925781  0.454619744
# 3   0.09164816  0.09164399  0.084908958
# 4   0.30611824  0.30607993  0.269004559
# 5  -0.21784237 -0.21782483 -0.204585702
# 6   0.13218973  0.13216699  0.109658578
# 7   0.24965870  0.24962693  0.221328409
# 8   0.62695324  0.62687745  0.555594457
# 9   0.36478667  0.36473406  0.314134816
# 10 -0.34503284 -0.34500076 -0.315831128
# 11 -0.11476342 -0.11475156 -0.106099807
# 12 -0.06966812 -0.06966015 -0.061974909
# 13 -0.03947601 -0.03947246 -0.034488419
# 14  0.44120157  0.44115244  0.397308957
# 15  0.02054657  0.02052406  0.001887644
# 16 -0.42789853 -0.42785114 -0.383567943
# 17  0.20116250  0.20113428  0.175680291
# 18  0.10358870  0.10357382  0.089780515
# 19 -0.27052886 -0.27049497 -0.239048662
# 20  0.29462234  0.29457974  0.255140387
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.