Changeset 40325 for trunk/Ohana/src/opihi/cmd.data/test/nnet.sh

Timestamp:

Jan 27, 2018, 1:09:32 PM (8 years ago)

Author:

eugene

Message:

updated nnet to include better init, L2 regularization, cross-entropy cost function

File:

• trunk/Ohana/src/opihi/cmd.data/test/nnet.sh (modified) (3 diffs)

trunk/Ohana/src/opihi/cmd.data/test/nnet.sh

@@ -161 +161 @@
   nnet apply t1 x0 x1 y0 y1
 
-  nnet create tn 2 2
-  nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta
+# -large-weight-initializer : revert to original implementation
+  nnet create tn 2 2 
+# nnet create tn 2 2 -large-weight-initializer
+
+# -quadratic-cost : revert to original implementation
+  nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost
+# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0
+# nnet train tn x0 x1 y0 y1 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 2.0
 
   nnet apply tn x0 x1 Y0 Y1
@@ -169 +175 @@
   set dy1 = y1 - Y1
 
-  vstat -q dy0
+  vstat dy0
   $yp = $MAX
   $ym = $MIN
 
-  vstat -q dy1
+  vstat dy1
   $yp = max ($MAX , $yp)
   $ym = min ($MIN , $ym)
@@ -181 +187 @@
 end
 
+macro test.bilevel
+  if ($0 != 4)
+    echo "USAGE: test.bilevel (Nmini) (Nepoch) (eta)"
+    break
+  end
+
+  local Nmini Nepoch eta
+  $Nmini  = $1
+  $Nepoch = $2
+  $eta    = $3
+
+  # create a nnet with just 2 inputs and 2 outputs
+  nnet create t1 3 4 3
+
+  # biases:
+  vlist v1 -0.2 0.0 0.2 0.4
+  vlist v2 -0.5 0.0 0.5
+
+  # weights
+  mcreate m1 3 4
+  m1[0][0] =  0.2
+  m1[1][0] = -0.2
+  m1[2][0] = -0.3
+
+  m1[0][1] = -0.6
+  m1[1][1] =  0.5
+  m1[2][1] = -0.1
+
+  m1[0][2] = -0.2
+  m1[1][2] =  0.2
+  m1[2][2] = -0.5
+
+  m1[0][3] =  0.4
+  m1[1][3] = -0.5
+  m1[2][3] = -0.7
+
+  mcreate m2 4 3
+  m2[0][0] =  0.8
+  m2[1][0] = -0.4
+  m2[2][0] = -0.5
+  m2[3][0] =  0.3
+
+  m2[0][1] = -0.2
+  m2[1][1] = -0.5
+  m2[2][1] =  0.1
+  m2[3][1] =  0.1
+
+  m2[0][2] =  0.2
+  m2[1][2] = -0.2
+  m2[2][2] =  0.5
+  m2[3][2] = -0.1
+
+  # set weights and biases
+  nnet set t1 m1 v1 m1 v1
+
+  # generate an input set spanning the range -5 to +5
+
+  create n 0 10000
+  set x0 = 10*rnd(n) - 5.0
+  set x1 = 10*rnd(n) - 5.0
+  set x2 = 10*rnd(n) - 5.0
+
+  # get output vectors the these input vectors
+  nnet apply t1 x0 x1 x2 y0 y1 y2
+
+# nnet create tn 3 4 3 -large-weight-initializer
+  nnet create tn 3 4 3
+
+# nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.1
+  nnet train tn x0 x1 x2 y0 y1 y2 -Nmini $Nmini -Nepoch $Nepoch -eta $eta -lambda 0.0 -quadratic-cost
+
+  nnet apply tn x0 x1 x2 Y0 Y1 Y2
+
+  set dy0 = y0 - Y0
+  set dy1 = y1 - Y1
+  set dy2 = y2 - Y2
+
+  vstat dy0
+  $yp = $MAX
+  $ym = $MIN
+
+  vstat dy1
+  $yp = max ($MAX , $yp)
+  $ym = min ($MIN , $ym)
+
+  vstat dy2
+  $yp = max ($MAX , $yp)
+  $ym = min ($MIN , $ym)
+# $yp = 0.02; $ym = -0.02
+
+  style -pt circle -sz 2; 
+  lim x0 $ym $yp; clear; box; 
+  plot -c blue  x0 dy0; 
+  plot -c red   x0 dy1
+  plot -c black x0 dy2
+end
+