remove dependency to 'kex'. Moved relevant stuff from kex to 'nn' or …

…'unsup'

CMakeLists.txt

@@ -3,7 +3,10 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.6 FATAL_ERROR)
 CMAKE_POLICY(VERSION 2.6)
 FIND_PACKAGE(Torch REQUIRED)
 
+SET(src init.c)
+
 SET(luasrc init.lua 
+    Diag.lua
     AutoEncoder.lua
     SparseAutoEncoder.lua
     LinearFistaL1.lua 
@@ -17,4 +20,7 @@ SET(luasrc init.lua
     kmeans.lua
     test/test_fista.lua)
 
-ADD_TORCH_PACKAGE(unsup ""  "${luasrc}" "Unsupervised Learning")
+INCLUDE_DIRECTORIES(${Torch_SOURCE_INCLUDES})
+ADD_TORCH_PACKAGE(unsup "${src}"  "${luasrc}" "Unsupervised Learning")
+
+TARGET_LINK_LIBRARIES(unsup luaT TH)

Diag.lua

@@ -0,0 +1,44 @@
+local Diag,parent = torch.class('nn.Diag','nn.Module')
+
+function Diag:__init(nFeature)
+   parent.__init(self)
+   self.weight = torch.Tensor(nFeature)
+   self.gradWeight = torch.Tensor(nFeature)
+
+   self:reset()
+end
+
+function Diag:reset(stdv)
+   self.weight:fill(1)
+end
+
+function Diag:updateOutput(input)
+   self.output:resizeAs(input):copy(input)
+   if input:dim() > 1 then
+      for i=1,input:size(1) do
+	 self.output[{{i}}]:mul(self.weight[i])
+      end
+   else
+      self.output:cmul(self.weight)
+   end
+   return self.output
+end
+
+function Diag:updateGradInput(input, gradOutput)
+   self.gradInput:resizeAs(gradOutput):copy(gradOutput)
+   if input:dim() > 1 then
+      for i=1,input:size(1) do
+	 self.gradInput[{{i}}]:mul(self.weight[i])
+      end
+   else
+      self.gradInput:cmul(self.weight)
+   end
+   return self.gradInput
+end
+
+function Diag:accGradParameters(input, gradOutput, scale)
+   for i=1,input:size(1) do
+      self.gradWeight[i] = self.gradWeight[i] + scale*gradOutput[{{i}}]:dot(input[{{i}}])
+   end
+end
+

demo/demo_data.lua

@@ -57,7 +57,7 @@ end
 
 function getdatacam(inputsize, std)
    require 'camera'
-   local frow = 60
+   local frow = 45
    local fcol = 80
    local gs = 5
    local cam = image.Camera{width=fcol,height=frow}
@@ -118,14 +118,17 @@ function getdatacam(inputsize, std)
 	 print('2',lvar:min(),lvar:max())
       end
 
-      lvar:apply(function (x) if x<0 then return 0 else return x end end)
+      --lvar:apply(function (x) if x<0 then return 0 else return x end end)
+      lvar[torch.lt(lvar,0)] = 0
       if data_verbose then
 	 print('2',lvar:min(),lvar:max())
       end
+
 
       local lstd = lvar
       lstd:sqrt()
-      lstd:apply(function (x) if x<1 then return 1 else return x end end)
+      --lstd:apply(function (x) if x<1 then return 1 else return x end end)
+      lstd[torch.lt(lstd,1)]=1
       if data_verbose then
 	 print('lstd',lstd:min(),lstd:max())
       end

demo/demo_fistaui.lua

@@ -25,7 +25,7 @@ cmd:option('-kernelsize', 9, 'size of convolutional kernels')
 cmd:option('-inputsize', 9, 'size of each input patch')
 cmd:option('-lambda', 1, 'sparsity coefficient')
 cmd:option('-datafile', 'tr-berkeley-N5K-M56x56-lcn.bin','Data set file')
-cmd:option('-eta',0.1,'learning rate')
+cmd:option('-eta',0.01,'learning rate')
 cmd:option('-momentum',0,'gradient momentum')
 cmd:option('-decay',0,'weigth decay')
 cmd:option('-maxiter',1000000,'max number of updates')
@@ -53,19 +53,21 @@ torch.manualSeed(params.seed)
 -- create the dataset
 if params.cam then
    data = getdatacam(params.inputsize)
+   params.dispupdate = 5
 else
-   if not paths.filep(datafile) then
+   if not paths.filep(params.datafile) then
       print('Datafile does not exist : ' .. params.datafile)
       print('You can get sample datafile from http://cs.nyu.edu/~koray/publis/code/tr-berkeley-N5K-M56x56-lcn.bin')
    end
    data = getdata(params.datafile, params.inputsize)
+   params.dispupdate = 100
 end
 
 -- creat unsup stuff
 mlp = unsup.LinearFistaL1(params.inputsize*params.inputsize, params.nfiltersout, params.lambda )
 
 -- do learrning rate hacks
-kex.nnhacks()
+--kex.nnhacks()
 
 function train(module,dataset)
 
@@ -78,8 +80,9 @@ function train(module,dataset)
       module:zeroGradParameters()
       module:updateGradInput(input, target)
       module:accGradParameters(input, target)
-      --print(module.D.gradWeight:sum())
+      --print(module.D.gradWeight:sum(),eta)
       module:updateParameters(eta)
+      module:normalize()
       return err, #h
    end
 
@@ -96,7 +99,7 @@ function train(module,dataset)
       err = err + serr
       iter = iter + siter
 
-      if math.fmod(t, 100) == 0 then
+      if math.fmod(t, params.dispupdate) == 0 then
          ww:gbegin()
          ww:showpage()
          ww:setfontsize(25)
@@ -105,7 +108,6 @@ function train(module,dataset)
          --print('plotting')
          image.display{win=ww,image=example[3],x=10,y=60,zoom=2, symmetric=true}
          image.display{win=ww,image=mlp.D.weight:transpose(1,2):unfold(2,9,9),padding=1,nrow=8,symetric=true,x=example[3]:size(2)*2+30, y=60,zoom=3}
-
          ww:show(string.format('%6.2f : %6.2f',example[3]:min(), example[3]:max()), 10, 60+example[3]:size(1)*2+5,100, 20)
          ww:show(string.format('%6.2f : %6.2f',mlp.D.weight:min(), mlp.D.weight:max()),example[3]:size(2)*2+30,60+120,100,20)
          ww:gend()

demo/demo_psd_conv.lua

@@ -1,5 +1,4 @@
 require 'unsup'
-require 'kex'
 require 'image'
 
 dofile 'demo_data.lua'
@@ -46,6 +45,7 @@ local params = cmd:parse(arg)
 if not params.hessian then
    error('convolutional psd runs much much much faster with psd')
 end
+nn.hessian.enable()
 
 if params.openmp or params.nThread > 1 then
    torch.setDefaultNumThreads(params.nThread)

generic/util.c

@@ -0,0 +1,41 @@
+#ifndef TH_GENERIC_FILE
+#define TH_GENERIC_FILE "generic/util.c"
+#else
+
+static int unsup_(shrinkage)(lua_State *L)
+{
+  real lambda = luaL_checknumber(L,2);
+  THTensor *tensor = luaT_checkudata(L,1, torch_Tensor);
+  luaL_argcheck(L, lambda >=0, 2, "Lambda should be non-negative");
+
+  if (lambda == 0) return 1;
+
+  TH_TENSOR_APPLY(real, tensor,
+		  if (*tensor_data > lambda)
+		  {
+		    *tensor_data -= lambda;
+		  }
+		  else if (*tensor_data < -lambda)
+		  {
+		    *tensor_data += lambda;
+		  }
+		  else
+		  {
+		    *tensor_data = 0;
+		  });
+  return 1;
+}
+
+static const struct luaL_Reg unsup_(util__) [] = {
+  {"shrinkage", unsup_(shrinkage)},
+  {NULL, NULL}
+};
+
+static void unsup_(util_init)(lua_State *L)
+{
+  luaT_pushmetatable(L, torch_Tensor);
+  luaL_register(L, NULL, unsup_(util__));
+  lua_pop(L,1);
+}
+
+#endif

init.c

@@ -0,0 +1,24 @@
+#include "TH.h"
+#include "luaT.h"
+
+#define torch_(NAME) TH_CONCAT_3(torch_, Real, NAME)
+#define torch_Tensor TH_CONCAT_STRING_3(torch.,Real,Tensor)
+#define unsup_(NAME) TH_CONCAT_3(unsup_, Real, NAME)
+#define torch_string_(NAME) TH_CONCAT_STRING_3(torch., Real, NAME)
+
+#include "generic/util.c"
+#include "THGenerateFloatTypes.h"
+
+DLL_EXPORT int luaopen_libunsup(lua_State *L)
+{
+  lua_newtable(L);
+  lua_pushvalue(L, -1);
+  lua_setfield(L, LUA_GLOBALSINDEX, "unsup");
+
+  unsup_Floatutil_init(L);
+  unsup_Doubleutil_init(L);
+
+  return 1;
+}
+
+

init.lua

@@ -1,11 +1,9 @@
-require 'torch'
 require 'nn'
-
-require 'kex'
 require 'optim'
+require 'libunsup'
 
-unsup = {}
-
+-- extra modules
+torch.include('unsup', 'Diag.lua')
 -- classes that implement algorithms
 torch.include('unsup', 'UnsupModule.lua')
 torch.include('unsup', 'AutoEncoder.lua')
@@ -18,3 +16,28 @@ torch.include('unsup', 'ConvPsd.lua')
 torch.include('unsup', 'UnsupTrainer.lua')
 torch.include('unsup', 'pca.lua')
 torch.include('unsup', 'kmeans.lua')
+
+local oldhessian = nn.hessian.enable
+function nn.hessian.enable()
+	oldhessian() -- enable Hessian usage
+	----------------------------------------------------------------------
+	-- Diag
+	----------------------------------------------------------------------
+	local accDiagHessianParameters = nn.hessian.accDiagHessianParameters
+	local updateDiagHessianInput = nn.hessian.updateDiagHessianInput
+	local updateDiagHessianInputPointWise = nn.hessian.updateDiagHessianInputPointWise
+	local initDiagHessianParameters = nn.hessian.initDiagHessianParameters
+
+	function nn.Diag.updateDiagHessianInput(self, input, diagHessianOutput)
+	   updateDiagHessianInput(self, input, diagHessianOutput, {'weight'}, {'weightSq'})
+	   return self.diagHessianInput
+	end
+
+	function nn.Diag.accDiagHessianParameters(self, input, diagHessianOutput)
+	   accDiagHessianParameters(self,input, diagHessianOutput, {'gradWeight'}, {'diagHessianWeight'})
+	end
+
+	function nn.Diag.initDiagHessianParameters(self)
+	   initDiagHessianParameters(self,{'gradWeight'},{'diagHessianWeight'})
+	end
+end