Changeset 40325 for trunk/Ohana/src/opihi/cmd.data/nnet_train.c

Timestamp:

Jan 27, 2018, 1:09:32 PM (8 years ago)

Author:

eugene

Message:

updated nnet to include better init, L2 regularization, cross-entropy cost function

File:

• trunk/Ohana/src/opihi/cmd.data/nnet_train.c (modified) (8 diffs)

trunk/Ohana/src/opihi/cmd.data/nnet_train.c

@@ -7 +7 @@
 void  nnet_reset_Nabla (Nnet *nnet);
 void  nnet_update_Nabla (Nnet *nnet);
-void  nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta);
+void  nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta, float lambda, int Ntrial);
 void  nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N);
-void  nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta);
+void  nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta, float lambda);
+
+static int QUADRATIC_COST = 0;
 
 int nnet_train (int argc, char **argv) {
@@ -34 +36 @@
     eta = atof (argv[N]);
     remove_argument (N, &argc, argv);
+  }
+
+  float lambda = 0.0; // XXX how do I set this? does it need to update?
+  if ((N = get_argument (argc, argv, "-lambda"))) {
+    remove_argument (N, &argc, argv);
+    lambda = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
+  QUADRATIC_COST = 0;
+  if ((N = get_argument (argc, argv, "-quadratic-cost"))) {
+    remove_argument (N, &argc, argv);
+    QUADRATIC_COST = 1;    
   }
 
@@ -121 +136 @@
       // for a given pass select seq elements pass*Nmini to pass*Nmini + Nmini - 1
       // update the weights and biases using the mini batch subset
-      nnet_descent_step (nnet, inVec, outVec, seq, pass, Nmini, eta);
+      nnet_descent_step (nnet, inVec, outVec, seq, pass, Nmini, eta, lambda);
       gprint (GP_ERR, "epoch %d of %d, pass %d of %d\n", epoch, Nepoch, pass, Npass);
     }
@@ -134 +149 @@
 
 // this recreates 'update_mini_batch' from http://neuralnetworksanddeeplearning.com/chap1.html
-void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta) {
+void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta, float lambda) {
 
   int Ntrial = inVec[0][0].Nelements;
@@ -151 +166 @@
   }
 
-  nnet_apply_Nabla (nnet, Nmini, eta);
+  nnet_apply_Nabla (nnet, Nmini, eta, lambda, Ntrial);
 }
 
@@ -183 +198 @@
         // NOTE: the (1/Npt) factor is pushed to the apply_Nabla step
         // nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
-        nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
+
+        if (QUADRATIC_COST) {
+          nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
+        } else {
+          nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]);
+        }
+
+        // for quadratic cost, delta = (svalue[L] - output) * sprime[L]
+        // for cross-entropy, delta = (svalue[L] - output)
+
       }
     } else {
@@ -218 +242 @@
 
 // support functions to loop over the Nabla entries
-void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta) {
+void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta, float lambda, int Ntrial) {
   for (int L = 1; L < nnet[0].Nlayer; L++) {
 
@@ -226 +250 @@
       for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
         int k = i + j*nnet[0].Nnodes[L-1];
-        nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k];
+        // nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k];
+        // with lambda > 0.0, we have L2 regularization.  if lambda = 0.0, we recover the default implementation
+        nnet[0].weight[L][k] = nnet[0].weight[L][k]*(1.0 - eta*lambda/Ntrial) - (eta / Nmini) * nnet[0].Nabla_w[L][k];
       }
     }