Index: trunk/Ohana/src/opihi/cmd.data/nnet_commands.c
===================================================================
--- trunk/Ohana/src/opihi/cmd.data/nnet_commands.c	(revision 40323)
+++ trunk/Ohana/src/opihi/cmd.data/nnet_commands.c	(revision 40325)
@@ -36,4 +36,12 @@
 int nnet_create (int argc, char **argv) {
 
+  int N;
+
+  int LargeWeightInit = FALSE;
+  if ((N = get_argument (argc, argv, "-large-weight-initializer"))) {
+    remove_argument (N, &argc, argv);
+    LargeWeightInit = TRUE;    
+  }
+
   if (argc < 4) {
     gprint (GP_ERR, "USAGE: nnet create (nnet) (Ninput) [Nnodes] [Nnodes] ... (Noutput)\n");
@@ -52,5 +60,5 @@
 
   // allocate the vector and matrix data arrays (and init)
-  CreateNnetData (nnet);
+  CreateNnetData (nnet, LargeWeightInit);
 
   return TRUE;
Index: trunk/Ohana/src/opihi/cmd.data/nnet_train.c
===================================================================
--- trunk/Ohana/src/opihi/cmd.data/nnet_train.c	(revision 40323)
+++ trunk/Ohana/src/opihi/cmd.data/nnet_train.c	(revision 40325)
@@ -7,7 +7,9 @@
 void  nnet_reset_Nabla (Nnet *nnet);
 void  nnet_update_Nabla (Nnet *nnet);
-void  nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta);
+void  nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta, float lambda, int Ntrial);
 void  nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N);
-void  nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta);
+void  nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta, float lambda);
+
+static int QUADRATIC_COST = 0;
 
 int nnet_train (int argc, char **argv) {
@@ -34,4 +36,17 @@
     eta = atof (argv[N]);
     remove_argument (N, &argc, argv);
+  }
+
+  float lambda = 0.0; // XXX how do I set this? does it need to update?
+  if ((N = get_argument (argc, argv, "-lambda"))) {
+    remove_argument (N, &argc, argv);
+    lambda = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
+  QUADRATIC_COST = 0;
+  if ((N = get_argument (argc, argv, "-quadratic-cost"))) {
+    remove_argument (N, &argc, argv);
+    QUADRATIC_COST = 1;    
   }
 
@@ -121,5 +136,5 @@
       // for a given pass select seq elements pass*Nmini to pass*Nmini + Nmini - 1
       // update the weights and biases using the mini batch subset
-      nnet_descent_step (nnet, inVec, outVec, seq, pass, Nmini, eta);
+      nnet_descent_step (nnet, inVec, outVec, seq, pass, Nmini, eta, lambda);
       gprint (GP_ERR, "epoch %d of %d, pass %d of %d\n", epoch, Nepoch, pass, Npass);
     }
@@ -134,5 +149,5 @@
 
 // this recreates 'update_mini_batch' from http://neuralnetworksanddeeplearning.com/chap1.html
-void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta) {
+void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta, float lambda) {
 
   int Ntrial = inVec[0][0].Nelements;
@@ -151,5 +166,5 @@
   }
 
-  nnet_apply_Nabla (nnet, Nmini, eta);
+  nnet_apply_Nabla (nnet, Nmini, eta, lambda, Ntrial);
 }
 
@@ -183,5 +198,14 @@
 	// NOTE: the (1/Npt) factor is pushed to the apply_Nabla step
 	// nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
-	nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
+
+	if (QUADRATIC_COST) {
+	  nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
+	} else {
+	  nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]);
+	}
+
+	// for quadratic cost, delta = (svalue[L] - output) * sprime[L]
+	// for cross-entropy, delta = (svalue[L] - output)
+
       }
     } else {
@@ -218,5 +242,5 @@
 
 // support functions to loop over the Nabla entries
-void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta) {
+void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta, float lambda, int Ntrial) {
   for (int L = 1; L < nnet[0].Nlayer; L++) {
 
@@ -226,5 +250,7 @@
       for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
 	int k = i + j*nnet[0].Nnodes[L-1];
-	nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k];
+	// nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k];
+	// with lambda > 0.0, we have L2 regularization.  if lambda = 0.0, we recover the default implementation
+	nnet[0].weight[L][k] = nnet[0].weight[L][k]*(1.0 - eta*lambda/Ntrial) - (eta / Nmini) * nnet[0].Nabla_w[L][k];
       }
     }
Index: trunk/Ohana/src/opihi/cmd.data/test/nnet.sh
===================================================================
--- trunk/Ohana/src/opihi/cmd.data/test/nnet.sh	(revision 40323)
+++ trunk/Ohana/src/opihi/cmd.data/test/nnet.sh	(revision 40325)
@@ -161,6 +161,12 @@
   nnet apply t1 x0 x1 y0 y1
 
-  nnet create tn 2 2
-  nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta
+# -large-weight-initializer : revert to original implementation
+  nnet create tn 2 2 
+# nnet create tn 2 2 -large-weight-initializer
+
+# -quadratic-cost : revert to original implementation
+  nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost
+# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0
+# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 2.0
 
   nnet apply tn x0 x1 Y0 Y1
@@ -169,9 +175,9 @@
   set dy1 = y1 - Y1
 
-  vstat -q dy0
+  vstat dy0
   $yp = $MAX
   $ym = $MIN
 
-  vstat -q dy1
+  vstat dy1
   $yp = max ($MAX , $yp)
   $ym = min ($MIN , $ym)
@@ -181,2 +187,99 @@
 end
 
+macro test.bilevel
+  if ($0 != 4)
+    echo "USAGE: test.bilevel (Nmini) (Nepoch) (eta)"
+    break
+  end
+
+  local Nmini Nepoch eta
+  $Nmini  = $1
+  $Nepoch = $2
+  $eta    = $3
+
+  # create a nnet with just 2 inputs and 2 outputs
+  nnet create t1 3 4 3
+
+  # biases:
+  vlist v1 -0.2 0.0 0.2 0.4
+  vlist v2 -0.5 0.0 0.5
+
+  # weights
+  mcreate m1 3 4
+  m1[0][0] =  0.2
+  m1[1][0] = -0.2
+  m1[2][0] = -0.3
+
+  m1[0][1] = -0.6
+  m1[1][1] =  0.5
+  m1[2][1] = -0.1
+
+  m1[0][2] = -0.2
+  m1[1][2] =  0.2
+  m1[2][2] = -0.5
+
+  m1[0][3] =  0.4
+  m1[1][3] = -0.5
+  m1[2][3] = -0.7
+
+  mcreate m2 4 3
+  m2[0][0] =  0.8
+  m2[1][0] = -0.4
+  m2[2][0] = -0.5
+  m2[3][0] =  0.3
+
+  m2[0][1] = -0.2
+  m2[1][1] = -0.5
+  m2[2][1] =  0.1
+  m2[3][1] =  0.1
+
+  m2[0][2] =  0.2
+  m2[1][2] = -0.2
+  m2[2][2] =  0.5
+  m2[3][2] = -0.1
+
+  # set weights and biases
+  nnet set t1 m1 v1 m1 v1
+
+  # generate an input set spanning the range -5 to +5
+
+  create n 0 10000
+  set x0 = 10*rnd(n) - 5.0
+  set x1 = 10*rnd(n) - 5.0
+  set x2 = 10*rnd(n) - 5.0
+
+  # get output vectors the these input vectors
+  nnet apply t1 x0 x1 x2 y0 y1 y2
+
+# nnet create tn 3 4 3 -large-weight-initializer
+  nnet create tn 3 4 3
+
+# nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.1
+  nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost
+
+  nnet apply tn x0 x1 x2 Y0 Y1 Y2
+
+  set dy0 = y0 - Y0
+  set dy1 = y1 - Y1
+  set dy2 = y2 - Y2
+
+  vstat dy0
+  $yp = $MAX
+  $ym = $MIN
+
+  vstat dy1
+  $yp = max ($MAX , $yp)
+  $ym = min ($MIN , $ym)
+
+  vstat dy2
+  $yp = max ($MAX , $yp)
+  $ym = min ($MIN , $ym)
+# $yp = 0.02; $ym = -0.02
+
+  style -pt circle -sz 2; 
+  lim x0 $ym $yp; clear; box; 
+  plot -c blue  x0 dy0; 
+  plot -c red   x0 dy1
+  plot -c black x0 dy2
+end
+