Changeset 40325 for trunk

Timestamp:

Jan 27, 2018, 1:09:32 PM (8 years ago)

Author:

eugene

Message:

updated nnet to include better init, L2 regularization, cross-entropy cost function

Location:

trunk/Ohana/src/opihi

Files:

• cmd.data/nnet_commands.c (modified) (2 diffs)
• cmd.data/nnet_train.c (modified) (8 diffs)
• cmd.data/test/nnet.sh (modified) (3 diffs)
• include/data.h (modified) (1 diff)
• lib.data/nnet.c (modified) (2 diffs)

trunk/Ohana/src/opihi/cmd.data/nnet_commands.c

@@ -36 +36 @@
 int nnet_create (int argc, char **argv) {
 
+  int N;
+
+  int LargeWeightInit = FALSE;
+  if ((N = get_argument (argc, argv, "-large-weight-initializer"))) {
+    remove_argument (N, &argc, argv);
+    LargeWeightInit = TRUE;    
+  }
+
   if (argc < 4) {
     gprint (GP_ERR, "USAGE: nnet create (nnet) (Ninput) [Nnodes] [Nnodes] ... (Noutput)\n");
@@ -52 +60 @@
 
   // allocate the vector and matrix data arrays (and init)
-  CreateNnetData (nnet);
+  CreateNnetData (nnet, LargeWeightInit);
 
   return TRUE;

trunk/Ohana/src/opihi/cmd.data/nnet_train.c

@@ -7 +7 @@
 void  nnet_reset_Nabla (Nnet *nnet);
 void  nnet_update_Nabla (Nnet *nnet);
-void  nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta);
+void  nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta, float lambda, int Ntrial);
 void  nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N);
-void  nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta);
+void  nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta, float lambda);
+
+static int QUADRATIC_COST = 0;
 
 int nnet_train (int argc, char **argv) {
@@ -34 +36 @@
     eta = atof (argv[N]);
     remove_argument (N, &argc, argv);
+  }
+
+  float lambda = 0.0; // XXX how do I set this? does it need to update?
+  if ((N = get_argument (argc, argv, "-lambda"))) {
+    remove_argument (N, &argc, argv);
+    lambda = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
+  QUADRATIC_COST = 0;
+  if ((N = get_argument (argc, argv, "-quadratic-cost"))) {
+    remove_argument (N, &argc, argv);
+    QUADRATIC_COST = 1;    
   }
 
@@ -121 +136 @@
       // for a given pass select seq elements pass*Nmini to pass*Nmini + Nmini - 1
       // update the weights and biases using the mini batch subset
-      nnet_descent_step (nnet, inVec, outVec, seq, pass, Nmini, eta);
+      nnet_descent_step (nnet, inVec, outVec, seq, pass, Nmini, eta, lambda);
       gprint (GP_ERR, "epoch %d of %d, pass %d of %d\n", epoch, Nepoch, pass, Npass);
     }
@@ -134 +149 @@
 
 // this recreates 'update_mini_batch' from http://neuralnetworksanddeeplearning.com/chap1.html
-void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta) {
+void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta, float lambda) {
 
   int Ntrial = inVec[0][0].Nelements;
@@ -151 +166 @@
   }
 
-  nnet_apply_Nabla (nnet, Nmini, eta);
+  nnet_apply_Nabla (nnet, Nmini, eta, lambda, Ntrial);
 }
 
@@ -183 +198 @@
         // NOTE: the (1/Npt) factor is pushed to the apply_Nabla step
         // nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
-        nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
+
+        if (QUADRATIC_COST) {
+          nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
+        } else {
+          nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]);
+        }
+
+        // for quadratic cost, delta = (svalue[L] - output) * sprime[L]
+        // for cross-entropy, delta = (svalue[L] - output)
+
       }
     } else {
@@ -218 +242 @@
 
 // support functions to loop over the Nabla entries
-void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta) {
+void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta, float lambda, int Ntrial) {
   for (int L = 1; L < nnet[0].Nlayer; L++) {
 
@@ -226 +250 @@
       for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
         int k = i + j*nnet[0].Nnodes[L-1];
-        nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k];
+        // nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k];
+        // with lambda > 0.0, we have L2 regularization.  if lambda = 0.0, we recover the default implementation
+        nnet[0].weight[L][k] = nnet[0].weight[L][k]*(1.0 - eta*lambda/Ntrial) - (eta / Nmini) * nnet[0].Nabla_w[L][k];
       }
     }

trunk/Ohana/src/opihi/cmd.data/test/nnet.sh

@@ -161 +161 @@
   nnet apply t1 x0 x1 y0 y1
 
-  nnet create tn 2 2
-  nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta
+# -large-weight-initializer : revert to original implementation
+  nnet create tn 2 2 
+# nnet create tn 2 2 -large-weight-initializer
+
+# -quadratic-cost : revert to original implementation
+  nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost
+# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0
+# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 2.0
 
   nnet apply tn x0 x1 Y0 Y1
@@ -169 +175 @@
   set dy1 = y1 - Y1
 
-  vstat -q dy0
+  vstat dy0
   $yp = $MAX
   $ym = $MIN
 
-  vstat -q dy1
+  vstat dy1
   $yp = max ($MAX , $yp)
   $ym = min ($MIN , $ym)
@@ -181 +187 @@
 end
 
+macro test.bilevel
+  if ($0 != 4)
+    echo "USAGE: test.bilevel (Nmini) (Nepoch) (eta)"
+    break
+  end
+
+  local Nmini Nepoch eta
+  $Nmini  = $1
+  $Nepoch = $2
+  $eta    = $3
+
+  # create a nnet with just 2 inputs and 2 outputs
+  nnet create t1 3 4 3
+
+  # biases:
+  vlist v1 -0.2 0.0 0.2 0.4
+  vlist v2 -0.5 0.0 0.5
+
+  # weights
+  mcreate m1 3 4
+  m1[0][0] =  0.2
+  m1[1][0] = -0.2
+  m1[2][0] = -0.3
+
+  m1[0][1] = -0.6
+  m1[1][1] =  0.5
+  m1[2][1] = -0.1
+
+  m1[0][2] = -0.2
+  m1[1][2] =  0.2
+  m1[2][2] = -0.5
+
+  m1[0][3] =  0.4
+  m1[1][3] = -0.5
+  m1[2][3] = -0.7
+
+  mcreate m2 4 3
+  m2[0][0] =  0.8
+  m2[1][0] = -0.4
+  m2[2][0] = -0.5
+  m2[3][0] =  0.3
+
+  m2[0][1] = -0.2
+  m2[1][1] = -0.5
+  m2[2][1] =  0.1
+  m2[3][1] =  0.1
+
+  m2[0][2] =  0.2
+  m2[1][2] = -0.2
+  m2[2][2] =  0.5
+  m2[3][2] = -0.1
+
+  # set weights and biases
+  nnet set t1 m1 v1 m1 v1
+
+  # generate an input set spanning the range -5 to +5
+
+  create n 0 10000
+  set x0 = 10*rnd(n) - 5.0
+  set x1 = 10*rnd(n) - 5.0
+  set x2 = 10*rnd(n) - 5.0
+
+  # get output vectors the these input vectors
+  nnet apply t1 x0 x1 x2 y0 y1 y2
+
+# nnet create tn 3 4 3 -large-weight-initializer
+  nnet create tn 3 4 3
+
+# nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.1
+  nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost
+
+  nnet apply tn x0 x1 x2 Y0 Y1 Y2
+
+  set dy0 = y0 - Y0
+  set dy1 = y1 - Y1
+  set dy2 = y2 - Y2
+
+  vstat dy0
+  $yp = $MAX
+  $ym = $MIN
+
+  vstat dy1
+  $yp = max ($MAX , $yp)
+  $ym = min ($MIN , $ym)
+
+  vstat dy2
+  $yp = max ($MAX , $yp)
+  $ym = min ($MIN , $ym)
+# $yp = 0.02; $ym = -0.02
+
+  style -pt circle -sz 2; 
+  lim x0 $ym $yp; clear; box; 
+  plot -c blue  x0 dy0; 
+  plot -c red   x0 dy1
+  plot -c black x0 dy2
+end
+

trunk/Ohana/src/opihi/include/data.h

@@ -261 +261 @@
 Nnet *FindNnet (char *name);
 Nnet *CreateNnet (char *name, int Nlayer);
-void CreateNnetData (Nnet *nnet);
+void CreateNnetData (Nnet *nnet, int LargeWeightInit);
 
 int DeleteNnet (Nnet *nnet);

trunk/Ohana/src/opihi/lib.data/nnet.c

@@ -143 +143 @@
 }
 
-void CreateNnetData (Nnet *nnet) {
+void CreateNnetData (Nnet *nnet, int LargeWeightInit) {
 
   ohana_gaussdev_init ();
@@ -168 +168 @@
     }
 
+    float sigma = LargeWeightInit ? 1.0 : 1.0 / sqrt(nnet[0].Nnodes[i-1]);
+
     ALLOCATE (nnet[0].weight[i], float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]);  // weight connected each node in the previous layer to the current layer (excludes input layer)
     for (int j = 0; j < nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]; j++) {
-      nnet[0].weight[i][j] = ohana_gaussdev_rnd (0.0, 1.0);
+      nnet[0].weight[i][j] = ohana_gaussdev_rnd (0.0, sigma);
     }