use constant weight matrix

[boofhead]

I think this must be trivial but I can't work to how to do it.

I would like to have one weight matrix manually entered and not trained.

I am using R

I have tried:
data <- mx.symbol.Convolution(data = data, weight=1:4, kernel=c(1,1), stride=c(1,1), pad = c(0,0), no.bias=TRUE, num.filter = 1L)

and
extWeight <- mx.symbol.Blockgrad(mx.symbol.Variable('extWeight'))
data <- mx.symbol.Convolution(data = data, weight=extWeight, kernel=c(1,1), stride=c(1,1), pad = c(0,0), no.bias=TRUE, num.filter = 1L)
with arg.params = list("extWeight" = 1:4) in the call to mx.model.Feedforward.create()

I don't think any of this is working. Can someone help me out?

[piiswrong]

in python this can be done with fixed_param_names in mx.mod.Module, not sure about R.

You probably have to manually specify grad_req={'weights_to_train':'write', 'weights_not_totrain':'null'} in symbol.simple_bind

[piiswrong]

@thirdwing

[thirdwing]

I have to say it is possible, but not an easy way.

[boofhead]

I did this which works but is maybe not the best way.

1. Include the parameters as mx.symbol.Variable
2. modifies the mx.model.Feedforward.create and train functions as below and used my new argument init.params to set the parameters.

thinNet.mx.model.FeedForward.create <- function (symbol, X, y = NULL, ctx = NULL, num.round = 10, optimizer = "sgd", 
                                                initializer = mx.init.uniform(0.01), eval.data = NULL, eval.metric = NULL, 
                                                epoch.end.callback = NULL, batch.end.callback = NULL, array.batch.size = 128, 
                                                array.layout = "auto", kvstore = "local", verbose = TRUE, 
                                                arg.params = NULL, aux.params = NULL, init.params = NULL, ...) {
  
  if (is.array(X) || is.matrix(X)) {
    if (array.layout == "auto") {
      array.layout <- mx.model.select.layout.train(X, y)
    }
    if (array.layout == "rowmajor") {
      X <- t(X)
    }
  }
  X <- mx.model.init.iter(X, y, batch.size = array.batch.size, 
                          is.train = TRUE)
  if (!X$iter.next()) {
    X$reset()
    if (!X$iter.next()) 
      stop("Empty input")
  }
  
  input.shape <- dim((X$value())$data)
  ndim <- length(input.shape)
  batchsize = input.shape[[ndim]]
  input.shape <- list(data = input.shape)
  init.params.name <- names(init.params)
  input.shape <- append(input.shape,
                        lapply(init.params,function(param){
                          dim(param)
                        }))
  
  params <- mx.model.init.params.rnn(symbol, input.shape, initializer, 
                                     mx.cpu())
  for (name in init.params.name) {
    params$arg.params[[name]] <- mx.nd.array(init.params[[name]])
  }
  if (!is.null(arg.params)) 
    params$arg.params <- arg.params
  if (!is.null(aux.params)) 
    params$aux.params <- aux.params
  if (is.null(ctx)) 
    ctx <- mx.ctx.default()
  if (is.mx.context(ctx)) {
    ctx <- list(ctx)
  }
  if (!is.list(ctx)) 
    stop("ctx must be mx.context or list of mx.context")
  if (is.character(optimizer)) {
    optimizer <- mx.opt.create(optimizer, rescale.grad = (1/batchsize), 
                               ...)
  }
  if (!is.null(eval.data) && !is.list(eval.data) && !is.mx.dataiter(eval.data)) {
    stop("The validation set should be either a mx.io.DataIter or a R list")
  }
  if (is.list(eval.data)) {
    if (is.null(eval.data$data) || is.null(eval.data$label)) {
      stop("Please provide the validation set as list(data=R.array, label=R.array)")
    }
    if (is.array(eval.data$data) || is.matrix(eval.data$data)) {
      if (array.layout == "auto") {
        array.layout <- mx.model.select.layout.train(eval.data$data, 
                                                     eval.data$label)
      }
      if (array.layout == "rowmajor") {
        eval.data$data <- t(eval.data$data)
      }
    }
    eval.data <- mx.model.init.iter(eval.data$data, eval.data$label, 
                                    batch.size = array.batch.size, is.train = TRUE)
  }
  kvstore <- mx.model.create.kvstore(kvstore, params$arg.params, 
                                     length(ctx), verbose = verbose)
  model <- thinNet.mx.model.train(symbol, ctx, input.shape, params$arg.params, 
                                 params$aux.params, 1, num.round, optimizer = optimizer, 
                                 train.data = X, eval.data = eval.data, metric = eval.metric, 
                                 epoch.end.callback = epoch.end.callback, batch.end.callback = batch.end.callback, 
                                 kvstore = kvstore, verbose = verbose)
  return(model)
}




thinNet.mx.model.train <- function (symbol, ctx, input.shape, arg.params, aux.params, begin.round, 
                                   end.round, optimizer, train.data, eval.data, metric, epoch.end.callback, 
                                   batch.end.callback, kvstore, verbose = TRUE) {
  
  ndevice <- length(ctx)
  if (verbose) 
    cat(paste0("Start training with ", ndevice, " devices\n"))
  sliceinfo <- mx.model.slice.shape(input.shape$data, ndevice)
  train.execs <- lapply(1:ndevice, function(i) {
    shape <- input.shape
    shape$data <- sliceinfo[[i]]$shape
    custom.mx.simple.bind(symbol, ctx = ctx[[i]], data = shape, 
                          grad.req = "write")
  })
  for (texec in train.execs) {
    mx.exec.update.arg.arrays(texec, arg.params, match.name = TRUE)
    mx.exec.update.aux.arrays(texec, aux.params, match.name = TRUE)
  }
  params.index <- as.integer(mx.util.filter.null(lapply(1:length(train.execs[[1]]$ref.grad.arrays), 
                                                        function(k) {
                                                          if (!is.null(train.execs[[1]]$ref.grad.arrays[[k]])) k else NULL
                                                        })))
  update.on.kvstore <- FALSE
  if (!is.null(kvstore) && kvstore$update.on.kvstore) {
    update.on.kvstore <- TRUE
    kvstore$set.optimizer(optimizer)
  }
  else {
    updaters <- lapply(1:ndevice, function(i) {
      mx.opt.get.updater(optimizer, train.execs[[i]]$ref.arg.arrays)
    })
  }
  if (!is.null(kvstore)) {
    kvstore$init(params.index, train.execs[[1]]$ref.arg.arrays[params.index])
  }
  input.names <- mx.model.check.arguments(symbol)
  for (iteration in begin.round:end.round) {
    nbatch <- 0
    if (!is.null(metric)) {
      train.metric <- metric$init()
    }
    while (train.data$iter.next()) {
      dlist <- train.data$value()
      slices <- lapply(1:ndevice, function(i) {
        s <- sliceinfo[[i]]
        ret <- list(data = mx.nd.slice(dlist$data, s$begin, 
                                       s$end), label = mx.nd.slice(dlist$label, s$begin, 
                                                                   s$end))
        return(ret)
      })
      for (i in 1:ndevice) {
        s <- slices[[i]]
        names(s) <- input.names
        mx.exec.update.arg.arrays(train.execs[[i]], s, 
                                  match.name = TRUE)
      }
      for (texec in train.execs) {
        mx.exec.forward(texec, is.train = TRUE)
      }
      out.preds <- lapply(train.execs, function(texec) {
        mx.nd.copyto(texec$ref.outputs[[1]], mx.cpu())
      })
      for (texec in train.execs) {
        mx.exec.backward(texec)
      }
      if (!is.null(kvstore)) {
        kvstore$push(params.index, lapply(train.execs, 
                                          function(texec) {
                                            texec$ref.grad.arrays[params.index]
                                          }), -params.index)
      }
      if (update.on.kvstore) {
        kvstore$pull(params.index, lapply(train.execs, 
                                          function(texec) {
                                            texec$ref.arg.arrays[params.index]
                                          }), -params.index)
      }
      else {
        if (!is.null(kvstore)) {
          kvstore$pull(params.index, lapply(train.execs, 
                                            function(texec) {
                                              texec$ref.grad.arrays[params.index]
                                            }), -params.index)
        }
        arg.blocks <- lapply(1:ndevice, function(i) {
          updaters[[i]](train.execs[[i]]$ref.arg.arrays, 
                        train.execs[[i]]$ref.grad.arrays)
        })
        for (i in 1:ndevice) {
          mx.exec.update.arg.arrays(train.execs[[i]], 
                                    arg.blocks[[i]], skip.null = TRUE)
        }
      }
      if (!is.null(metric)) {
        for (i in 1:ndevice) {
          train.metric <- metric$update(slices[[i]]$label, 
                                        out.preds[[i]], train.metric)
        }
      }
      nbatch <- nbatch + 1
      if (!is.null(batch.end.callback)) {
        batch.end.callback(iteration, nbatch, environment())
      }
    }
    train.data$reset()
    if (!is.null(metric)) {
      result <- metric$get(train.metric)
      if (verbose) 
        cat(paste0("[", iteration, "] Train-", result$name, 
                   "=", result$value, "\n"))
    }
    if (!is.null(eval.data)) {
      if (!is.null(metric)) {
        eval.metric <- metric$init()
      }
      while (eval.data$iter.next()) {
        dlist <- eval.data$value()
        slices <- lapply(1:ndevice, function(i) {
          s <- sliceinfo[[i]]
          ret <- list(data = mx.nd.slice(dlist$data, 
                                         s$begin, s$end), label = mx.nd.slice(dlist$label, 
                                                                              s$begin, s$end))
          return(ret)
        })
        for (i in 1:ndevice) {
          s <- slices[[i]]
          names(s) <- input.names
          mx.exec.update.arg.arrays(train.execs[[i]], 
                                    s, match.name = TRUE)
        }
        for (texec in train.execs) {
          mx.exec.forward(texec, is.train = FALSE)
        }
        out.preds <- lapply(train.execs, function(texec) {
          mx.nd.copyto(texec$ref.outputs[[1]], mx.cpu())
        })
        if (!is.null(metric)) {
          for (i in 1:ndevice) {
            eval.metric <- metric$update(slices[[i]]$label, 
                                         out.preds[[i]], eval.metric)
          }
        }
      }
      eval.data$reset()
      if (!is.null(metric)) {
        result <- metric$get(eval.metric)
        if (verbose) 
          cat(paste0("[", iteration, "] Validation-", 
                     result$name, "=", result$value, "\n"))
      }
    }
    else {
      eval.metric <- NULL
    }
    model <- mx.model.extract.model(symbol, train.execs)
    epoch_continue <- TRUE
    if (!is.null(epoch.end.callback)) {
      epoch_continue <- epoch.end.callback(iteration, 0, 
                                           environment(), verbose = verbose)
    }
    if (!epoch_continue) {
      break
    }
  }
  return(model)
}