Changeset 33115

Timestamp:

Jan 18, 2012, 8:31:56 AM (15 years ago)

Author:

eugene

Message:

adding int and time types to read function

Location:

branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data

Files:

• read_vectors.c (modified) (9 diffs)
• test/read.sh (added)
• test/read.t1.dat (added)
• test/read.t2.dat (added)
• test/read.t3.dat (added)
• test/read.t4.dat (added)

branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/read_vectors.c

@@ -21 +21 @@
 }
 
+// vector types
+enum {COLTYPE_NONE, COLTYPE_FLT, COLTYPE_INT, COLTYPE_TIME};
+
 int read_vectors (int argc, char **argv) {
   
-    int i, j, Nskip, Nvec, *col, done, status, IsCSV, VERBOSE;
-  int Nbytes, nbytes, Nstart, NELEM, N, nread;
+  int TimeFormat;
+  time_t TimeReference;
+  int i, j, Nskip, Narg, Nvec, *col, IsCSV, VERBOSE;
+  int Nbytes, Nstart, NELEM, Nelem, nread, *coltype;
   char *colstr, *c0, *c1, *buffer, *extname;
-  double value;
   Vector **vec;
 
   /* auto-sense table type */
-  if ((N = get_argument (argc, argv, "-fits"))) {
-    remove_argument (N, &argc, argv);
-    extname = strcreate (argv[N]);
+  if ((Narg = get_argument (argc, argv, "-fits"))) {
+    remove_argument (Narg, &argc, argv);
+    extname = strcreate (argv[Narg]);
     if (extname == (char *) NULL) return (FALSE);
-    remove_argument (N, &argc, argv);
-    status = read_table_vectors (argc, argv, extname);
+    remove_argument (Narg, &argc, argv);
+    int status = read_table_vectors (argc, argv, extname);
     free (extname);
     return (status);
@@ -41 +45 @@
 
   Nskip = 0;
-  if ((N = get_argument (argc, argv, "-skip"))) {
-    remove_argument (N, &argc, argv);
-    Nskip = atof (argv[N]);
-    remove_argument (N, &argc, argv);
+  if ((Narg = get_argument (argc, argv, "-skip"))) {
+    remove_argument (Narg, &argc, argv);
+    Nskip = atof (argv[Narg]);
+    remove_argument (Narg, &argc, argv);
   }
 
   IsCSV = FALSE;
-  if ((N = get_argument (argc, argv, "-csv"))) {
-    remove_argument (N, &argc, argv);
+  if ((Narg = get_argument (argc, argv, "-csv"))) {
+    remove_argument (Narg, &argc, argv);
     IsCSV = TRUE;
   }
 
   VERBOSE = FALSE;
-  if ((N = get_argument (argc, argv, "-v"))) {
-    remove_argument (N, &argc, argv);
+  if ((Narg = get_argument (argc, argv, "-v"))) {
+    remove_argument (Narg, &argc, argv);
     VERBOSE = TRUE;
   }
@@ -65 +69 @@
   /* read name N name N  */
 
+  // do this only optionally?
+  GetTimeFormat (&TimeReference, &TimeFormat);
+
   if (f == (FILE *) NULL) {
     gprint (GP_ERR, "no open file for read\n");
@@ -71 +78 @@
   fseeko (f, 0LL, SEEK_SET);
 
-//  if (IsCSV) {
-//    status = read_vectors_csv (argc, argv, Nskip, f);
-//    return status;
-//  }
-
   Nvec = (argc - 1) / 2;
   ALLOCATE (vec, Vector *, Nvec);
   ALLOCATE (col, int, Nvec);
+  ALLOCATE (coltype, int, Nvec);
 
   for (i = 0; i < Nvec; i++) {
+
+    // interpret the column names including type flags
+    // XXX review the grammar before releasing this : is foo:type best, or is something else needed?
+    // Note the conflict wrt list entries
+    // the name may be of the form foo:type, where type may be one of : int, float, time
+    
+    coltype[i] = COLTYPE_FLT;
+    char *colname = argv[2*i + 1];
+    char *ptr = strchr (colname, ':');
+    if (ptr) {
+      // split out colname and type
+      *ptr = 0;
+      ptr ++;
+      if (!ptr) goto bad_colname;
+      coltype[i] = COLTYPE_NONE;
+      if (!strcasecmp(ptr, "float")) { coltype[i] = COLTYPE_FLT; }
+      if (!strcasecmp(ptr, "int"))   { coltype[i] = COLTYPE_INT; }
+      if (!strcasecmp(ptr, "time"))  { coltype[i] = COLTYPE_TIME; }
+      if (!coltype[i]) goto bad_colname;
+    }
+
     if ((vec[i] = SelectVector (argv[2*i + 1], ANYVECTOR, TRUE)) == NULL) {
       gprint (GP_ERR, "USAGE: read name N name N ...\n");
@@ -87 +111 @@
       return (FALSE);    
     }
-    // XXX we could allow flags (eg, N.i) to specify INT vs FLT types vectors...
+
     colstr = argv[2*i+2];
     for (j = 0; j < strlen (colstr); j++) {
@@ -119 +143 @@
   NELEM = 1000;
   for (i = 0; i < Nvec; i++) {
-    ResetVector (vec[i], OPIHI_FLT, NELEM);
+    if (coltype[i] == COLTYPE_INT) {
+      ResetVector (vec[i], OPIHI_INT, NELEM);
+    } else {
+      // note that COLTYPE_TIME is a type of float
+      ResetVector (vec[i], OPIHI_FLT, NELEM);
+    }
   }
   
@@ -128 +157 @@
   }
 
-  Nstart = 0;
-  N = 0;
-  done = FALSE;
-  while (!done) {
+  // we have a working buffer read from the file. we parse the lines in the working buffer
+  // until we reach the last chunk without an EOL char.  at that point, we shift the start
+  // of the last (partial) line to the start of the buffer and re-fill.
+
+  Nstart = 0; // location of the last valid byte in the buffer (start filling here)
+  Nelem = 0; // number of valid rows read (vector elements)
+  while (TRUE) {
     Nbytes = 0x10000 - Nstart;
     bzero (&buffer[Nstart], Nbytes);
@@ -139 +171 @@
       break;
     }
-    if (nread == 0) break;
-    nbytes = nread + Nstart;
+    if (nread == 0) break; // end of the file
+    // nbytes = nread + Nstart;
     
-    status = TRUE;
-    c0 = buffer; 
-    while (status) {
-      c1 = strchr (c0, '\n');
+    int bufferStatus = TRUE; 
+    c0 = buffer; // c0 always marks the start of a line
+    while (bufferStatus) {
+      c1 = strchr (c0, '\n'); // find the end of this current line
       if (c1 == (char *) NULL) {
         Nstart = strlen (c0);
         memmove (buffer, c0, Nstart);
-        status = FALSE;
-      } else {
-        *c1 = 0;
-      }      
-      if ((*c0 != '#') && (*c0 != '!')) {
-        for (i = 0; (i < Nvec) && status; i++) {
+        bufferStatus = FALSE;
+        continue;
+      }
+      *c1 = 0; // mark the end of the line 
+
+      if (*c0 == '#') { c0 = c1 + 1; continue; }
+      if (*c0 == '!') { c0 = c1 + 1; continue; }
+
+      // parse the vectors in this line.  this code is a bit inefficient: each column
+      // requires a separate pass through the line.
+
+      int lineStatus = TRUE;
+      for (i = 0; i < Nvec; i++) {
+        int ivalue;
+        double dvalue;
+        time_t tvalue;
+        int readStatus;
+        // need to make the if cases for coltype[i]
+        switch (coltype[i]) {
+          case COLTYPE_INT:
+            readStatus = IsCSV ? iparse_csv (&ivalue, col[i], c0) : iparse (&ivalue, col[i], c0);
+            vec[i][0].elements.Int[Nelem] = readStatus ? ivalue : 0;
+            break;
+          case COLTYPE_FLT:
+            readStatus = IsCSV ? dparse_csv (&dvalue, col[i], c0) : dparse (&dvalue, col[i], c0);
+            vec[i][0].elements.Flt[Nelem] = readStatus ? dvalue : NAN;
+            break;
+          case COLTYPE_TIME:
+            readStatus = IsCSV ? tparse_csv (&tvalue, col[i], c0) : tparse (&tvalue, col[i], c0);
+            dvalue = TimeValue (tvalue, TimeReference, TimeFormat);
+            vec[i][0].elements.Flt[Nelem] = readStatus ? dvalue : NAN;
+            break;
+        }
+        if (!readStatus && VERBOSE) {
           if (IsCSV) {
-            status = dparse_csv (&value, col[i], c0);
+            gprint (GP_ERR, "suspect field: %d (%s) in %s\n", col[i], argv[2*i+2], c0);
           } else {
-            status = dparse (&value, col[i], c0);
+            gprint (GP_ERR, "suspect field: %d in %s\n", col[i], c0);
           }
-          if (status) {
-              vec[i][0].elements.Flt[N] = value;
-          } else {
-              vec[i][0].elements.Flt[N] = NAN;
-              if (VERBOSE) {
-                  if (IsCSV) {
-                      gprint (GP_ERR, "suspect field: %d (%s) in %s\n", col[i], argv[2*i+2], c0);
-                  } else {
-                      gprint (GP_ERR, "suspect field: %d in %s\n", col[i], c0);
-                  }
-              }
-          }
-        }
-        if (status) {
-            N++;
-        } else {
-            if (VERBOSE && FALSE) {
-                char temp[32];
-                strncpy (temp, c0, 32);
-                temp[31] = 0;
-                gprint (GP_ERR, "skip line %s\n\n", temp);
-            }
-        }
-      }
-      c0 = c1 + 1;
-      if (N == NELEM) {
+        }
+        lineStatus &= readStatus;
+      }
+      if (!lineStatus && VERBOSE) {
+        char temp[32];
+        strncpy (temp, c0, 32);
+        temp[31] = 0;
+        gprint (GP_ERR, "skip line %s\n\n", temp);
+      }
+      Nelem ++;
+      if (Nelem == NELEM) {
         NELEM += 1000;
         for (i = 0; i < Nvec; i++) {
-          REALLOCATE (vec[i][0].elements.Flt, opihi_flt, NELEM);
-        }
-      }
+          if (coltype[i] == COLTYPE_INT) {
+            REALLOCATE (vec[i][0].elements.Int, opihi_int, NELEM);
+          } else {
+            REALLOCATE (vec[i][0].elements.Flt, opihi_flt, NELEM);
+          }
+        }
+      }
+      c0 = c1 + 1;
     }
   }
   for (i = 0; i < Nvec; i++) {
-    REALLOCATE (vec[i][0].elements.Flt, opihi_flt, MAX (N,1));
-    vec[i][0].Nelements = N;
+    if (coltype[i] == COLTYPE_INT) {
+      REALLOCATE (vec[i][0].elements.Int, opihi_int, MAX (Nelem,1));
+    } else {
+      REALLOCATE (vec[i][0].elements.Flt, opihi_flt, MAX (Nelem,1));
+    }
+    vec[i][0].Nelements = Nelem;
   }
   
@@ -399 +453 @@
   return (TRUE);
 }
-
-# if (0)
-int read_vectors_csv (int argc, char **argv, int Nskip, FILE *f) {
-  
-  int i, j, Nvec, *col, done, status;
-  int Nbytes, nbytes, Nstart, NELEM, N, nread;
-  char *colstr, *c0, *c1, *buffer, *extname;
-  double value;
-  Vector **vec;
-
-  Nvec = (argc - 1) / 2;
-  ALLOCATE (vec, Vector *, Nvec);
-  ALLOCATE (col, int, Nvec);
-
-  for (i = 0; i < Nvec; i++) {
-    if ((vec[i] = SelectVector (argv[2*i + 1], ANYVECTOR, TRUE)) == NULL) {
-      gprint (GP_ERR, "USAGE: read name N name N ...\n");
-      free (vec);
-      free (col);
-      return (FALSE);    
-    }
-    // XXX we could allow flags (eg, N.i) to specify INT vs FLT types vectors...
-    colstr = argv[2*i+2];
-    for (j = 0; j < strlen (colstr); j++) {
-      if (!isdigit(colstr[j])) {
-        gprint (GP_ERR, "USAGE: read name N name N ...\n");
-        free (vec);
-        free (col);
-        return (FALSE);    
-      }
-    }
-    col[i] = atof (colstr);
-  }
-
-  // currently, all read vectors are forced to be type FLT
-  NELEM = 1000;
-  for (i = 0; i < Nvec; i++) {
-    ResetVector (vec[i], OPIHI_FLT, NELEM);
-  }
-  
-  ALLOCATE (buffer, char, 0x10001);
-  bzero (buffer, 0x10001);
-  for (i = 0; i < Nskip; i++) {
-    scan_line (f, buffer);
-  }
-
-  Nstart = 0;
-  N = 0;
-  done = FALSE;
-  while (!done) {
-    Nbytes = 0x10000 - Nstart;
-    bzero (&buffer[Nstart], Nbytes);
-    nread = fread (&buffer[Nstart], 1, Nbytes, f);
-    if (ferror (f)) {
-      perror ("error reading data file");
-      break;
-    }
-    if (nread == 0) break;
-    nbytes = nread + Nstart;
-    
-    status = TRUE;
-    c0 = buffer; 
-    while (status) {
-      c1 = strchr (c0, '\n');
-      if (c1 == (char *) NULL) {
-        Nstart = strlen (c0);
-        memmove (buffer, c0, Nstart);
-        status = FALSE;
-      } else {
-        *c1 = 0;
-      }      
-      if ((*c0 != '#') && (*c0 != '!')) {
-        for (i = 0; (i < Nvec) && status; i++) {
-          status = dparse_csv (&value, col[i], c0);
-          vec[i][0].elements.Flt[N] = value;
-          if (!status) vec[i][0].elements.Flt[N] = NAN;
-        }
-        if (status) N++;
-      }
-      c0 = c1 + 1;
-      if (N == NELEM) {
-        NELEM += 1000;
-        for (i = 0; i < Nvec; i++) {
-          REALLOCATE (vec[i][0].elements.Flt, opihi_flt, NELEM);
-        }
-      }
-        
-    }
-  }
-  for (i = 0; i < Nvec; i++) {
-    REALLOCATE (vec[i][0].elements.Flt, opihi_flt, MAX (N,1));
-    vec[i][0].Nelements = N;
-  }
-  
-  free (vec);
-  free (col);
-  free (buffer);
-  return (TRUE);
-
-}
-
-# endif