IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 40319


Ignore:
Timestamp:
Jan 23, 2018, 10:11:50 AM (8 years ago)
Author:
eugene
Message:

working on nnet

Location:
trunk/Ohana/src/opihi
Files:
2 added
8 edited

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/opihi/cmd.data/Makefile

    r40291 r40319  
    9999$(SRC)/medimage_commands.$(ARCH).o \
    100100$(SRC)/mset.$(ARCH).o           \
    101 $(SRC)/needles.$(ARCH).o                \
     101$(SRC)/needles.$(ARCH).o        \
     102$(SRC)/nnet.$(ARCH).o           \
     103$(SRC)/nnet_commands.$(ARCH).o  \
     104$(SRC)/nnet_train.$(ARCH).o     \
     105$(SRC)/nnet_apply.$(ARCH).o     \
    102106$(SRC)/peak.$(ARCH).o           \
    103107$(SRC)/periodogram.$(ARCH).o    \
    … …  
    189193$(INC)/external.h \
    190194$(INC)/shell.h \
     195$(INC)/data.h \
    191196$(INC)/dvomath.h \
    192197$(INC)/display.h

  • trunk/Ohana/src/opihi/cmd.data/init.c

    r40291 r40319  
    8989int mset             PROTO((int, char **));
    9090int needles          PROTO((int, char **));
     91int nnet_command     PROTO((int, char **));
    9192int peak             PROTO((int, char **));
    9293int periodogram      PROTO((int, char **));
    … …  
    270271  {1, "imset",        mset,             "insert a vector in an image"},
    271272  {1, "needles",      needles,          "plot vectors needles"},
     273  {1, "nnet",         nnet_command,     "Neural Network commands"},
    272274  {1, "parity",       parity,           "set image parity"},
    273275  {1, "peak",         peak,             "find vector peak in range"},

  • trunk/Ohana/src/opihi/cmd.data/nnet.c

    r40317 r40319  
    11# include "data.h"
    2 
    3 int nnet_list (int argc, char **argv);
    4 int nnet_create (int argc, char **argv);
    5 int nnet_delete (int argc, char **argv);
    6 int nnet_train (int argc, char **argv);
    7 int nnet_apply (int argc, char **argv);
    8 int nnet_show (int argc, char **argv);
    92
    103static Command nnet_commands[] = {
    114  {1, "list",     nnet_list,     "list nnets"},
    12   {1, "init",     nnet_init,     "initialize a nnet"},
    135  {1, "delete",   nnet_delete,   "delete a nnet"},
    146  {1, "show",     nnet_show,     "display nnet values"},
    157  {1, "create",   nnet_create,   "create a nnet"},
     8  {1, "set",      nnet_set,      "set nnet node values"},
     9  {1, "get",      nnet_get,      "get nnet node values"},
    1610  {1, "train",    nnet_train,    "train nnet on a set of data"},
    1711  {1, "apply",    nnet_apply,    "apply nnet to a set of data"},
    … …  
    2519    gprint (GP_ERR, "USAGE: nnet (command)\n");
    2620    gprint (GP_ERR, "    nnet list          : list nnets\n");
    27     gprint (GP_ERR, "    nnet init   (nnet) : removes all pages from nnet\n");
    2821    gprint (GP_ERR, "    nnet delete (nnet) : delete a nnet\n");
    2922    gprint (GP_ERR, "    nnet show   (nnet) : show values for a nnet\n");
    3023    gprint (GP_ERR, "    nnet create (nnet) (Ninput) [Nnodes] [Nnodes] ... (Noutput) : create a nnet\n");
     24    gprint (GP_ERR, "    nnet set    (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     25    gprint (GP_ERR, "    nnet get    (nnet) [weights] [biases] ... [weights] [biases] : get nnet weights (images) and biases (vectors)\n");
    3126    gprint (GP_ERR, "    nnet train  (nnet) [input] [input] ... [output] [output] ... : train nnet on data from a set of vectors\n");
    3227    gprint (GP_ERR, "    nnet apply  (nnet) [input] [input] ... [output] [output] ... : apply nnet to input data and generate output\n");

  • trunk/Ohana/src/opihi/cmd.data/nnet_commands.c

    r40317 r40319  
    1212}
    1313
    14 /** Fix These ...
    15 int nnet_init (int argc, char **argv) {
    16 
     14int nnet_delete (int argc, char **argv) {
     15
     16# if (0)
    1717  int status;
    1818  Nnet *nnet;
    1919
    2020  if (argc != 2) {
    21     gprint (GP_ERR, "USAGE: nnet init (nnet)\n");
    22     return FALSE;
    23   }
    24 
    25   nnet = FindNnet (argv[1]);
    26   if (nnet != NULL) {
    27       status = DeleteNnet (nnet);
    28       if (!status) abort ();
    29   }
    30 
    31   CreateNnet (argv[1]);
    32   return TRUE;
    33 }
    34 
    35 int nnet_delete (int argc, char **argv) {
    36 
    37   int status;
    38   Nnet *nnet;
    39 
    40   if (argc != 2) {
    4121    gprint (GP_ERR, "USAGE: nnet delete (nnet)\n");
    4222    return FALSE;
    … …  
    5131  status = DeleteNnet (nnet);
    5232  if (!status) abort ();
     33# endif
     34
    5335  return TRUE;
    5436}
    … …  
    5638int nnet_show (int argc, char **argv) {
    5739
     40# if (0)
    5841  Nnet *nnet;
    5942
    … …  
    7053
    7154  ListPages (nnet);
    72   return TRUE;
    73 }
    74 **/
     55# endif
     56
     57  return TRUE;
     58}
    7559
    7660int nnet_create (int argc, char **argv) {
    … …  
    9781}
    9882
     83int nnet_set (int argc, char **argv) {
     84
     85  Buffer *matrix;
     86  Vector *vector;
     87
     88  if (argc < 5) {
     89    gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     90    return FALSE;
     91  }
     92
     93  Nnet *nnet = FindNnet (argv[1]);
     94  if (nnet == NULL) {
     95    gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]);
     96    return FALSE;
     97  }
     98
     99  // compare argc and nnet[0].Nlayer
     100  if (argc - 2 != nnet[0].Nlayer * 2) {
     101    gprint (GP_ERR, "ERROR: invalid number of arguments\n");
     102    gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     103    return FALSE;
     104  }
     105
     106  // check for existence of each matrix & vector (and check sizes)
     107  for (int L = 1; L < nnet[0].Nlayer; L++) {
     108    int Narg = 2*(L - 1) + 2;
     109    if ((matrix = SelectBuffer (argv[Narg + 0], OLDBUFFER, FALSE)) == NULL) {
     110      gprint (GP_ERR, "unknown input image %s\n", argv[Narg + 0]);
     111      gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     112      return (FALSE);   
     113    }
     114    if ((vector = SelectVector (argv[Narg + 1], OLDVECTOR, FALSE)) == NULL) {
     115      gprint (GP_ERR, "unknown input vector %s\n", argv[Narg + 1]);
     116      gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     117      return (FALSE);   
     118    }
     119
     120    if (vector->Nelements != nnet[0].Nnodes[L]) {
     121      gprint (GP_ERR, "invalid bias vector length %s (%d vs %d)\n", argv[Narg + 1], vector->Nelements, nnet[0].Nnodes[L]);
     122      gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     123    }
     124
     125    if (matrix->matrix.Naxis[0] != nnet[0].Nnodes[L-1]) {
     126      gprint (GP_ERR, "invalid weight image size for %s (Nx = %d vs %d)\n", argv[Narg + 0], (int) matrix->matrix.Naxis[0], nnet[0].Nnodes[L-1]);
     127      gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     128    }
     129
     130    if (matrix->matrix.Naxis[1] != nnet[0].Nnodes[L]) {
     131      gprint (GP_ERR, "invalid weight image size for %s (Ny = %d vs %d)\n", argv[Narg + 0], (int) matrix->matrix.Naxis[1], nnet[0].Nnodes[L]);
     132      gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     133    }
     134
     135  }
     136
     137  for (int L = 1; L < nnet[0].Nlayer; L++) {
     138
     139    int Narg = 2*(L - 1) + 2;
     140    matrix = SelectBuffer (argv[Narg + 0], OLDBUFFER, FALSE); // I've checked the argv entry above so I do not need to check again...
     141    vector = SelectVector (argv[Narg + 1], OLDVECTOR, FALSE);
     142
     143    opihi_flt *vecvalue = vector[0].elements.Flt;
     144    for (int i = 0; i < nnet[0].Nnodes[L]; i++) {
     145      nnet[0].biases[L][i] = vecvalue[i];
     146    }
     147
     148    float *matvalue = (float *) matrix[0].matrix.buffer;
     149    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
     150      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
     151        int k = i + j*nnet[0].Nnodes[L-1];
     152        nnet[0].weight[L][k] = matvalue[k];
     153      }
     154    }
     155 
     156  }
     157
     158  return TRUE;
     159}
     160
     161int nnet_get (int argc, char **argv) {
     162
     163  Buffer *matrix;
     164  Vector *vector;
     165
     166  if (argc < 5) {
     167    gprint (GP_ERR, "USAGE: nnet get (nnet) [weights] [biases] ... [weights] [biases] : get nnet weights (images) and biases (vectors)\n");
     168    return FALSE;
     169  }
     170
     171  Nnet *nnet = FindNnet (argv[1]);
     172  if (nnet == NULL) {
     173    gprint (GP_ERR, "nnet %s not found, create it first\n", argv[1]);
     174    return FALSE;
     175  }
     176
     177  // compare argc and nnet[0].Nlayer
     178  if (argc - 2 != nnet[0].Nlayer * 2) {
     179    gprint (GP_ERR, "ERROR: invalid number of arguments\n");
     180    gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     181    return FALSE;
     182  }
     183
     184  // check for existence of each matrix & vector (and check sizes)
     185  for (int L = 1; L < nnet[0].Nlayer; L++) {
     186    int Narg = 2*(L - 1) + 2;
     187    if ((matrix = SelectBuffer (argv[Narg + 0], ANYBUFFER, FALSE)) == NULL) {
     188      gprint (GP_ERR, "unknown input image %s\n", argv[Narg + 0]);
     189      gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     190      return (FALSE);   
     191    }
     192    if ((vector = SelectVector (argv[Narg + 1], ANYVECTOR, FALSE)) == NULL) {
     193      gprint (GP_ERR, "unknown input vector %s\n", argv[Narg + 1]);
     194      gprint (GP_ERR, "USAGE: nnet set (nnet) [weights] [biases] ... [weights] [biases] : set nnet weights (images) and biases (vectors)\n");
     195      return (FALSE);   
     196    }
     197  }
     198
     199  for (int L = 1; L < nnet[0].Nlayer; L++) {
     200
     201    int Narg = 2*(L - 1) + 2;
     202    matrix = SelectBuffer (argv[Narg + 0], OLDBUFFER, FALSE); // I've checked the argv entry above so I do not need to check again...
     203    vector = SelectVector (argv[Narg + 1], OLDVECTOR, FALSE);
     204
     205    ResetVector (vector, OPIHI_FLT, nnet[0].Nnodes[L]);
     206    ResetBuffer (matrix, nnet[0].Nnodes[L-1], nnet[0].Nnodes[L], -32, 0.0, 1.0);
     207
     208    opihi_flt *vecvalue = vector[0].elements.Flt;
     209    for (int i = 0; i < nnet[0].Nnodes[L]; i++) {
     210      vecvalue[i] = nnet[0].biases[L][i];
     211    }
     212
     213    float *matvalue = (float *) matrix[0].matrix.buffer;
     214    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
     215      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
     216        int k = i + j*nnet[0].Nnodes[L-1];
     217        matvalue[k] = nnet[0].weight[L][k];
     218      }
     219    }
     220 
     221  }
     222
     223  return TRUE;
     224}
     225

  • trunk/Ohana/src/opihi/cmd.data/nnet_train.c

    r40318 r40319  
    11# include "data.h"
    2 void sortseq (float *X, int *Y, int N);
     2
     3// these are local only
     4float nnet_sigmoid (float value);
     5void  nnet_sortseq (float *X, int *Y, int N);
     6int   nnet_onelayer (Nnet *nnet, int L);
     7void  nnet_reset_Nabla (Nnet *nnet);
     8void  nnet_update_Nabla (Nnet *nnet);
     9void  nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta);
     10void  nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N);
     11void  nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta);
    312
    413int nnet_train (int argc, char **argv) {
     14
     15  int N;
    516
    617  int Nepoch = 10;
    … …  
    1829  }
    1930 
     31  float eta = 1.0; // XXX how do I set this? does it need to update?
     32
    2033  if (argc < 4) {
    2134    gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n");
    … …  
    97110    // generate a random sequence : used to select random mini batches
    98111    for (int i = 0; i < Ntrial; i++) { seq[i] = i; rnd[i] = drand48(); }
    99     sortseq (rnd, seq, Ntrial);
     112    nnet_sortseq (rnd, seq, Ntrial);
    100113
    101114    int Npass = Ntrial / Nmini;
    … …  
    116129
    117130// this recreates 'update_mini_batch' from http://neuralnetworksanddeeplearning.com/chap1.html
    118 void nnet_descent_step (NNet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta) {
     131void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta) {
    119132
    120133  int Ntrial = inVec[0][0].Nelements;
    … …  
    122135  nnet_reset_Nabla (nnet);
    123136
    124   for (i = 0; (i < Nmini) && (pass*Nmini + i < Ntrial); i++) {
     137  for (int i = 0; (i < Nmini) && (pass*Nmini + i < Ntrial); i++) {
    125138
    126139    // N is the element of the mini batch on which we are currently operating
    127     N = seq[pass*Nmini + i];
     140    int N = seq[pass*Nmini + i];
    128141
    129142    // backprop generates a dNabla_b, dNabla_w pair for the element N of the input and output vectors
    … …  
    136149}
    137150
    138 void nnet_backprop (NNet *nnet, Vector **inVec, Vector **outVec, int N) {
     151void nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N) {
    139152
    140153  // start with the input values
    141 
    142   // store the input values for this row (trial) in the input value vector
    143154  int Nlayer   = nnet[0].Nlayer;
    144   int Ninput   = nnet[0].Nnodes[0];
    145   int Noutput  = nnet[0].Nnodes[Nlayer - 1];  // number of input + output nodes
    146 
    147   for (int j = 0; j < Ninput; j++) {
     155
     156  // store the input values for this row (trial) in the input vector "svalue[0]"
     157  for (int j = 0; j < nnet[0].Nnodes[0]; j++) {
    148158    nnet[0].svalue[0][j] = inVec[j][0].elements.Flt[N];
    149159  }
    … …  
    153163  // activation = sigmoid (z)
    154164
     165  // feedforward operates on the vector saved in svalue[0]
     166  // feedforward saves the zvalues [w * input + bias], svalues [sigmoid(z)], sprimes [sigmoid'(z)] as it runs
    155167  nnet_feedforward (nnet);
    156168
    157   // backward pass (note for now these are vector operations:
    158   for (L = Nlayer - 1; L > 0; L--) {
    159 
    160     // sp is array of same size as zvalue for each layer
    161     // sp = sigmoid_prime (nnet[0].zvalue[L]); sprime is precalculated in feedforward
     169  // backward pass
     170  for (int L = Nlayer - 1; L > 0; L--) {
    162171
    163172    if (L == Nlayer - 1) {
    164173      // starting point uses cost_derivative to compare last svalue set with truth output
    165       // delta = cost_derivative(nnet[0].svalue[L], outVec, N) * sprime[L];
     174      // delta = cost_derivative(svalue[L], output) * sprime[L];
    166175      for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
    167         nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
     176        // cost_derivative (svalue[L][j], output[j]) = svalue[L][j] - output[j]
     177        // (because cost = (1/2)(svalue[L][j] - output[j])^2)
     178        // NOTE: the (1/Npt) factor is pushed to the apply_Nabla step
     179        // nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
     180        nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
    168181      }
    169182    } else {
    170       // delta = (delta DOT transpose(weights[L+1])) * sp;
     183      // delta = DOT(delta, transpose(weight[L+1])) * sprime;
    171184      for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
    172         tmpdelta = 0;
     185        float tmpdelta = 0.0;
    173186        for (int i = 0; i < nnet[0].Nnodes[L+1]; i++) {
    174           int k = j + i*nnet[0].Nnodes[L];
    175           tmpdelta += nnet[0].weight[L][k] * nnet[0].delta[L+1][i]; // XXX check on the index values
     187          int k = j + i*nnet[0].Nnodes[L]; // note order of (i,j) : j is [L+1] direction
     188          tmpdelta += nnet[0].weight[L][k] * nnet[0].delta[L+1][i];
    176189        }
    177190        nnet[0].delta[L][j] = tmpdelta * nnet[0].sprime[L][j];
    … …  
    179192    }                                         
    180193
    181     // UPDATE
    182     Nabla_b[L] = delta;
     194    // NOTE on weight array: for a given layer, L, the matrix weight[L] maps the nodes in
     195    // the previous layer (L-1) to those in L: Nnodes[L-1] -> Nnodes[L].  This is a matrix
     196    // with dimensions (Nnodes[L-1] x Nnodes[L]).  The Nnodes[L-1] is the fast dimension,
     197    // so for an element (i,j), the index k = i + j*Nnodes[L-1].   
     198
     199    // Nabla_b[L] = delta;
     200    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
     201      nnet[0]. Nabla_b[L][j] = nnet[0].delta[L][j];
     202    }
    183203   
    184     // Nabla_w[L] = delta DOT transpose(nnet[0].svalue[L-1]);
     204    // Nabla_w[L] = DOT(delta, transpose(svalue[L-1]));
    185205    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
    186206      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
    187         int k = i + j*nnet[0].Nnodes[n-1];
    188         nnet[0]. Nabla_w[n][k] = nnet[0].svalue[L-1][i] * nnet[0].delta[L][j];
    189       }
    190     }
    191 
     207        int k = i + j*nnet[0].Nnodes[L-1];
     208        nnet[0]. Nabla_w[L][k] = nnet[0].svalue[L-1][i] * nnet[0].delta[L][j];
     209      }
     210    }
    192211  }
    193212}
    194213
    195214// support functions to loop over the Nabla entries
    196 void nnet_apply_Nabla (NNet *nnet, int Nmini, float eta) {
    197   for (int n = 1; n < nnet[0].Nlayer; n++) {
    198 
    199     for (int j = 0; j < nnet[0].Nnodes[n]; j++) {
    200       nnet[0].biases[n][j] -= (eta / Nmini) * nnet[0].Nabla_b[n][j];
    201 
    202       for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) {
    203         int k = i + j*nnet[0].Nnodes[n-1];
    204         nnet[0].weights[n][k] -= (eta / Nmini) * nnet[0].Nabla_w[n][k];
    205       }
    206     }
    207   }
    208 }
    209 void nnet_update_Nabla (NNet *nnet) {
    210   for (int n = 1; n < nnet[0].Nlayer; n++) {
    211 
    212     for (int j = 0; j < nnet[0].Nnodes[n]; j++) {
    213       nnet[0]. Nabla_b[n][j] += nnet[0].dNabla_b[n][j];
    214 
    215       for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) {
    216         int k = i + j*nnet[0].Nnodes[n-1];
    217         nnet[0]. Nabla_w[n][k] += nnet[0].dNabla_w[n][k];
    218       }
    219     }
    220   }
    221 }
    222 void nnet_reset_Nabla (NNet *nnet) {
    223   for (int n = 1; n < nnet[0].Nlayer; n++) {
    224 
    225     for (int j = 0; j < nnet[0].Nnodes[n]; j++) {
    226 
    227       nnet[0]. Nabla_b[n][j] = 0;
    228       nnet[0].dNabla_b[n][j] = 0;
    229 
    230       for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) {
    231         int k = i + j*nnet[0].Nnodes[n-1];
    232         nnet[0]. Nabla_w[n][k] = 0;
    233         nnet[0].dNabla_w[n][k] = 0;
     215void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta) {
     216  for (int L = 1; L < nnet[0].Nlayer; L++) {
     217
     218    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
     219      nnet[0].biases[L][j] -= (eta / Nmini) * nnet[0].Nabla_b[L][j];
     220
     221      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
     222        int k = i + j*nnet[0].Nnodes[L-1];
     223        nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k];
     224      }
     225    }
     226  }
     227}
     228void nnet_update_Nabla (Nnet *nnet) {
     229  for (int L = 1; L < nnet[0].Nlayer; L++) {
     230
     231    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
     232      nnet[0]. Nabla_b[L][j] += nnet[0].dNabla_b[L][j];
     233
     234      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
     235        int k = i + j*nnet[0].Nnodes[L-1];
     236        nnet[0]. Nabla_w[L][k] += nnet[0].dNabla_w[L][k];
     237      }
     238    }
     239  }
     240}
     241void nnet_reset_Nabla (Nnet *nnet) {
     242  for (int L = 1; L < nnet[0].Nlayer; L++) {
     243
     244    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
     245
     246      nnet[0]. Nabla_b[L][j] = 0;
     247      nnet[0].dNabla_b[L][j] = 0;
     248
     249      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
     250        int k = i + j*nnet[0].Nnodes[L-1];
     251        nnet[0]. Nabla_w[L][k] = 0;
     252        nnet[0].dNabla_w[L][k] = 0;
    234253      }
    235254    }
    … …  
    238257
    239258// the input values must already be copied to the input layer svalue[]
    240 void nnet_feedforward (NNet *nnet) {
     259void nnet_feedforward (Nnet *nnet) {
    241260
    242261  for (int i = 1; i < nnet[0].Nlayer; i++) {
    … …  
    246265}
    247266
    248 // calcularte z, sigmoid(z) for each layer (z = w*value + bias)
    249 int nnet_onelayer (NNet *nnet, int n) {
    250 
    251   if (n < 1) return FALSE; // abort here?
    252   if (n >= nnet[0].Nlayer) return FALSE; // abort here?
    253 
    254   // evaluating a single layer [n], n > 0, n < Nlayer:
    255   int Ninput   = nnet[0].Nnodes[n - 1];
    256   int Noutput  = nnet[0].Nnodes[n];
    257 
    258   // input layer is [n-1], output layer is [n]
     267// calculate z, sigmoid(z) for each layer (z = w*value + bias)
     268int nnet_onelayer (Nnet *nnet, int L) {
     269
     270  if (L < 1) return FALSE; // abort here?
     271  if (L >= nnet[0].Nlayer) return FALSE; // abort here?
     272
     273  // evaluating a single layer [L], n > 0, n < Nlayer:
     274  int Ninput   = nnet[0].Nnodes[L - 1];
     275  int Noutput  = nnet[0].Nnodes[L];
     276
     277  // input layer is [L-1], output layer is [L]
    259278  for (int j = 0; j < Noutput; j ++) {
    260279    float sum = 0;
    … …  
    262281      // weight matrix order is (0, 1, ... Ninput-1, Ninput, Ninput + 1, ... Ninput * Noutput - 1)
    263282      int k = j * Ninput + i;
    264       sum += nnet[0].weight[n][k]*nnet[0].svalue[n-1][i];
    265     }
    266     sum += nnet[0].biases[n][j];
    267     nnet[0].zvalue[n][j] = sum;
    268     nnet[0].svalue[n][j] = nnet_sigmoid(sum);
    269     nnet[0].sprime[n][j] = nnet[0].svalue[n][j] * (1 - nnet[0].svalue[n][j]);
     283      sum += nnet[0].weight[L][k]*nnet[0].svalue[L-1][i];
     284    }
     285    sum += nnet[0].biases[L][j];
     286    nnet[0].zvalue[L][j] = sum;
     287    nnet[0].svalue[L][j] = nnet_sigmoid(sum);
     288    nnet[0].sprime[L][j] = nnet[0].svalue[L][j] * (1 - nnet[0].svalue[L][j]);
    270289    // note that d sigmoid / dz = sigmoid * (1 - sigmoid)
    271290  }
    … …  
    274293
    275294float nnet_sigmoid (float value) {
    276   return 1.0 / (1.0 + exp(-sum));
    277 }
    278 
    279 void sortseq (float *X, int *Y, int N) {
     295  return 1.0 / (1.0 + exp(-value));
     296}
     297
     298void nnet_sortseq (float *X, int *Y, int N) {
    280299
    281300# define SWAPFUNC(A,B){ float ftmp; int itmp; \

  • trunk/Ohana/src/opihi/include/data.h

    r40291 r40319  
    5050  float **buffers;
    5151} MedImageType;
     52
     53/*** typedef structs used by the Neural Network functions (nnet_*) ***/
     54typedef struct {
     55  char *name;
     56  int Nlayer; // Nlayers = input layer + output layer + hidden layers
     57  int *Nnodes; // number of nodes per layer
     58  float **weight; // a matrix between each layer
     59  float **biases; // a vector for each layer
     60
     61  float **zvalue; // a vector of z values for each layer (= w*input + b)
     62  float **svalue; // a vector of s values for each layer (= sigmoid(z))
     63  float **sprime; //
     64  float **delta;  //
     65
     66  float ** Nabla_b; // a vector of Nabla_b values for each layer
     67  float **dNabla_b; // a vector of Nabla_b values for each layer
     68
     69  float ** Nabla_w; // a matrix of Nabla_w values for each layer
     70  float **dNabla_w; // a matrix of Nabla_w values for each layer
     71} Nnet;
    5272
    5373void InitData (void);
    … …  
    229249int GetKapaChannelFromString (char *string);
    230250
     251/*** Neural Network functions (nnet_*) ***/
     252
     253// in lib.data/nnet.c:
     254void InitNnets ();
     255void InitNnetData (Nnet *nnet, char *name);
     256
     257void FreeNnets ();
     258void FreeNnetData (Nnet *nnet);
     259
     260Nnet *GetNnet (int where);
     261Nnet *FindNnet (char *name);
     262Nnet *CreateNnet (char *name, int Nlayer);
     263void CreateNnetData (Nnet *nnet);
     264
     265int DeleteNnet (Nnet *nnet);
     266void ListNnets ();
     267
     268// in cmd.data/nnet.c:
     269int nnet_command (int argc, char **argv);
     270
     271// in cmd.data/nnet_commands.c:
     272int nnet_init (int argc, char **argv);
     273int nnet_list (int argc, char **argv);
     274int nnet_delete (int argc, char **argv);
     275int nnet_show (int argc, char **argv);
     276int nnet_create (int argc, char **argv);
     277int nnet_set (int argc, char **argv);
     278int nnet_get (int argc, char **argv);
     279
     280// in cmd.data/nnet_train.c:
     281int nnet_train (int argc, char **argv);
     282void nnet_feedforward (Nnet *nnet);
     283
     284// in cmd.data/nnet_apply.c:
     285int nnet_apply (int argc, char **argv);
     286
    231287# endif

  • trunk/Ohana/src/opihi/lib.data/Makefile

    r39242 r40319  
    2020$(SDIR)/fft.$(ARCH).o                   \
    2121$(SDIR)/svdcmp.$(ARCH).o                \
    22 $(SDIR)/svdcmp_bond_new.$(ARCH).o               \
     22$(SDIR)/svdcmp_bond_new.$(ARCH).o       \
    2323$(SDIR)/bracket.$(ARCH).o               \
    2424$(SDIR)/spline.$(ARCH).o                \
    … …  
    2727$(SDIR)/mrqmin.$(ARCH).o                \
    2828$(SDIR)/mrq2dmin.$(ARCH).o              \
     29$(SDIR)/nnet.$(ARCH).o                  \
    2930$(SDIR)/precess.$(ARCH).o               \
    3031$(SDIR)/starfuncs.$(ARCH).o             \

  • trunk/Ohana/src/opihi/lib.data/nnet.c

    r40317 r40319  
    11# include "data.h"
    2 
    3 typedef struct {
    4   char *name;
    5   int Nlayers; // Nlayers = input layer + output layer + hidden layers
    6   int *Nnodes; // number of nodes per layer
    7   float **weights; // a matrix between each layer
    8   float **biases; // a vector for each layer
    9 
    10   float **zvalue; // a vector of z values for each layer (= w*input + b)
    11   float **svalue; // a vector of s values for each layer (= sigmoid(z))
    12   float **sprime; //
    13   float **delta;  //
    14 
    15   float ** Nabla_b; // a vector of Nabla_b values for each layer
    16   float **dNabla_b; // a vector of Nabla_b values for each layer
    17 
    18   float ** Nabla_w; // a matrix of Nabla_w values for each layer
    19   float **dNabla_w; // a matrix of Nabla_w values for each layer
    20 } NNet;
    212
    223static Nnet **nnets  = NULL; /* nnet to store the list of all nnets */
    … …  
    3415  int i;
    3516
    36   for (i = 0; i < Nnnets; i++) {
    37     FreeNnet (nnets[i]);
     17  if (!nnets) InitNnets();
     18
     19  for (i = 0; i < Nnnets; i++) {
     20    FreeNnetData (nnets[i]);
     21    free (nnets[i]);
    3822  }
    3923  free (nnets);
    … …  
    4630  nnet[0].name = strcreate (name);
    4731
    48   nnet[0].Nlayers = 0;
     32  nnet[0].Nlayer  = 0;
    4933  nnet[0].Nnodes  = NULL;
    50   nnet[0].weights = NULL;
     34  nnet[0].weight  = NULL;
    5135  nnet[0].biases  = NULL;
    52   nnet[0].avalue  = NULL;
     36  nnet[0].svalue  = NULL;
    5337  nnet[0].zvalue  = NULL;
     38  nnet[0].sprime  = NULL;
     39  nnet[0].delta   = NULL;
    5440
    5541  nnet[0]. Nabla_b  = NULL;
    … …  
    6450
    6551    free (nnet[0].name);
    66     for (i = 0; i < nnet[0].Nlayers; i++) {
    67       free (nnet[0].weights[i]);
     52    for (i = 0; i < nnet[0].Nlayer; i++) {
     53      free (nnet[0].weight[i]);
    6854      free (nnet[0].biases[i]);
    6955      free (nnet[0].svalue[i]);
    … …  
    7864    }
    7965    free (nnet[0].Nnodes);
    80     free (nnet[0].weights);
     66    free (nnet[0].weight);
    8167    free (nnet[0].biases);
    8268    free (nnet[0].svalue);
    … …  
    9480Nnet *GetNnet (int where) {
    9581
     82  if (!nnets) InitNnets();
     83
    9684  if (where < 0) where += Nnnets;
    9785  if (where < 0) return NULL;
    … …  
    10593  int i;
    10694
     95  if (!nnets) InitNnets();
     96
    10797  for (i = 0; i < Nnnets; i++) {
    10898    if (!strcmp (nnets[i][0].name, name)) {
    … …  
    113103}
    114104
    115 /* make a new named nnet with Nlayers */
     105/* make a new named nnet with Nlayer */
    116106Nnet *CreateNnet (char *name, int Nlayer) {
    117107
    118108  int N;
    119109  Nnet *nnet;
     110
     111  if (!nnets) InitNnets();
    120112
    121113  nnet = FindNnet (name);
    122114  if (nnet != NULL) {
    123115    FreeNnetData (nnet);
    124     InitNnetData (nnet);
     116    InitNnetData (nnet, name);
    125117    return (nnet);
    126118  }
    … …  
    134126  nnet[0].Nlayer = Nlayer;
    135127  ALLOCATE (nnet[0].Nnodes, int, Nlayer);
    136   ALLOCATE (nnet[0].weights, float *, Nlayer);
     128  ALLOCATE (nnet[0].weight, float *, Nlayer);
    137129  ALLOCATE (nnet[0].biases, float *, Nlayer);
    138130  ALLOCATE (nnet[0].svalue, float *, Nlayer);
    … …  
    155147  ohana_gaussdev_init ();
    156148
    157   for (int i = 1; i < Nlayer; i++) {
     149  // NOTE : none of these elements are used for the first layer (the input layer)
     150  // EXCEPT svalue[0]
     151 
     152  nnet[0].weight[0] = NULL;
     153  nnet[0].biases[0] = NULL;
     154  nnet[0].zvalue[0] = NULL;
     155  nnet[0].sprime[0] = NULL;
     156  nnet[0].delta [0] = NULL;
     157  nnet[0]. Nabla_b[0] = NULL;
     158  nnet[0].dNabla_b[0] = NULL;
     159  nnet[0]. Nabla_w[0] = NULL;
     160  nnet[0].dNabla_w[0] = NULL;
     161
     162  ALLOCATE (nnet[0].svalue[0], float, nnet[0].Nnodes[0]);
     163
     164  for (int i = 1; i < nnet[0].Nlayer; i++) {
    158165    ALLOCATE (nnet[0].biases[i], float, nnet[0].Nnodes[i]);  // biases for each node in the hidden and output layers only
    159166    for (int j = 0; j < nnet[0].Nnodes[i]; j++) {
    … …  
    161168    }
    162169
    163     ALLOCATE (nnet[0].weights[i], float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]);  // weights connected each node in the previous layer to the current layer (excludes input layer)
     170    ALLOCATE (nnet[0].weight[i], float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]);  // weight connected each node in the previous layer to the current layer (excludes input layer)
    164171    for (int j = 0; j < nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]; j++) {
    165       nnet[0].weights[i][j] = ohana_gaussdev_rnd (0.0, 1.0);
     172      nnet[0].weight[i][j] = ohana_gaussdev_rnd (0.0, 1.0);
    166173    }
    167174
    … …  
    171178    ALLOCATE (nnet[0].delta [i], float, nnet[0].Nnodes[i]);  // vectors for holding results / values for each node in the input, hidden, output layers
    172179
    173     ALLOCATE (nnet[0]. Nabla_b, float, nnet[0].Nnodes[i]);
    174     ALLOCATE (nnet[0].dNabla_b, float, nnet[0].Nnodes[i]);
    175 
    176     ALLOCATE (nnet[0]. Nabla_w, float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]);
    177     ALLOCATE (nnet[0].dNabla_w, float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]);
     180    ALLOCATE (nnet[0]. Nabla_b[i], float, nnet[0].Nnodes[i]);
     181    ALLOCATE (nnet[0].dNabla_b[i], float, nnet[0].Nnodes[i]);
     182    ALLOCATE (nnet[0]. Nabla_w[i], float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]);
     183    ALLOCATE (nnet[0].dNabla_w[i], float, nnet[0].Nnodes[i-1]*nnet[0].Nnodes[i]);
    178184  }
    179185}
    … …  
    183189
    184190  int i, N, NNNETS_2;
     191
     192  if (!nnets) InitNnets();
    185193
    186194  /* find nnet in nnet list */
    … …  
    204212  }
    205213
    206   FreeNnet (nnet);
     214  FreeNnetData (nnet);
     215  free (nnet);
    207216  return (TRUE);
    208217}
    … …  
    213222  int i, j;
    214223
     224  if (!nnets) InitNnets();
     225
    215226  for (i = 0; i < Nnnets; i++) {
    216227    gprint (GP_ERR, "%-15s :", nnets[i][0].name);
    217     for (j = 0; j < nnets[i][0].Nlayers; i++) {
     228    for (j = 0; j < nnets[i][0].Nlayer; j++) {
    218229      gprint (GP_ERR, " %3d", nnets[i][0].Nnodes[j]);
    219230    }
Note: See TracChangeset for help on using the changeset viewer.