Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/read_vectors.c
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/read_vectors.c	(revision 33114)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/read_vectors.c	(revision 33115)
@@ -21,19 +21,23 @@
 }
 
+// vector types
+enum {COLTYPE_NONE, COLTYPE_FLT, COLTYPE_INT, COLTYPE_TIME};
+
 int read_vectors (int argc, char **argv) {
   
-    int i, j, Nskip, Nvec, *col, done, status, IsCSV, VERBOSE;
-  int Nbytes, nbytes, Nstart, NELEM, N, nread;
+  int TimeFormat;
+  time_t TimeReference;
+  int i, j, Nskip, Narg, Nvec, *col, IsCSV, VERBOSE;
+  int Nbytes, Nstart, NELEM, Nelem, nread, *coltype;
   char *colstr, *c0, *c1, *buffer, *extname;
-  double value;
   Vector **vec;
 
   /* auto-sense table type */
-  if ((N = get_argument (argc, argv, "-fits"))) {
-    remove_argument (N, &argc, argv);
-    extname = strcreate (argv[N]);
+  if ((Narg = get_argument (argc, argv, "-fits"))) {
+    remove_argument (Narg, &argc, argv);
+    extname = strcreate (argv[Narg]);
     if (extname == (char *) NULL) return (FALSE);
-    remove_argument (N, &argc, argv);
-    status = read_table_vectors (argc, argv, extname);
+    remove_argument (Narg, &argc, argv);
+    int status = read_table_vectors (argc, argv, extname);
     free (extname);
     return (status);
@@ -41,19 +45,19 @@
 
   Nskip = 0;
-  if ((N = get_argument (argc, argv, "-skip"))) {
-    remove_argument (N, &argc, argv);
-    Nskip = atof (argv[N]);
-    remove_argument (N, &argc, argv);
+  if ((Narg = get_argument (argc, argv, "-skip"))) {
+    remove_argument (Narg, &argc, argv);
+    Nskip = atof (argv[Narg]);
+    remove_argument (Narg, &argc, argv);
   }
 
   IsCSV = FALSE;
-  if ((N = get_argument (argc, argv, "-csv"))) {
-    remove_argument (N, &argc, argv);
+  if ((Narg = get_argument (argc, argv, "-csv"))) {
+    remove_argument (Narg, &argc, argv);
     IsCSV = TRUE;
   }
 
   VERBOSE = FALSE;
-  if ((N = get_argument (argc, argv, "-v"))) {
-    remove_argument (N, &argc, argv);
+  if ((Narg = get_argument (argc, argv, "-v"))) {
+    remove_argument (Narg, &argc, argv);
     VERBOSE = TRUE;
   }
@@ -65,4 +69,7 @@
   /* read name N name N  */
 
+  // do this only optionally?
+  GetTimeFormat (&TimeReference, &TimeFormat);
+
   if (f == (FILE *) NULL) {
     gprint (GP_ERR, "no open file for read\n");
@@ -71,14 +78,31 @@
   fseeko (f, 0LL, SEEK_SET);
 
-//  if (IsCSV) {
-//    status = read_vectors_csv (argc, argv, Nskip, f);
-//    return status;
-//  }
-
   Nvec = (argc - 1) / 2;
   ALLOCATE (vec, Vector *, Nvec);
   ALLOCATE (col, int, Nvec);
+  ALLOCATE (coltype, int, Nvec);
 
   for (i = 0; i < Nvec; i++) {
+
+    // interpret the column names including type flags
+    // XXX review the grammar before releasing this : is foo:type best, or is something else needed?
+    // Note the conflict wrt list entries
+    // the name may be of the form foo:type, where type may be one of : int, float, time
+    
+    coltype[i] = COLTYPE_FLT;
+    char *colname = argv[2*i + 1];
+    char *ptr = strchr (colname, ':');
+    if (ptr) {
+      // split out colname and type
+      *ptr = 0;
+      ptr ++;
+      if (!ptr) goto bad_colname;
+      coltype[i] = COLTYPE_NONE;
+      if (!strcasecmp(ptr, "float")) { coltype[i] = COLTYPE_FLT; }
+      if (!strcasecmp(ptr, "int"))   { coltype[i] = COLTYPE_INT; }
+      if (!strcasecmp(ptr, "time"))  { coltype[i] = COLTYPE_TIME; }
+      if (!coltype[i]) goto bad_colname;
+    }
+
     if ((vec[i] = SelectVector (argv[2*i + 1], ANYVECTOR, TRUE)) == NULL) {
       gprint (GP_ERR, "USAGE: read name N name N ...\n");
@@ -87,5 +111,5 @@
       return (FALSE);    
     }
-    // XXX we could allow flags (eg, N.i) to specify INT vs FLT types vectors...
+
     colstr = argv[2*i+2];
     for (j = 0; j < strlen (colstr); j++) {
@@ -119,5 +143,10 @@
   NELEM = 1000;
   for (i = 0; i < Nvec; i++) {
-    ResetVector (vec[i], OPIHI_FLT, NELEM);
+    if (coltype[i] == COLTYPE_INT) {
+      ResetVector (vec[i], OPIHI_INT, NELEM);
+    } else {
+      // note that COLTYPE_TIME is a type of float
+      ResetVector (vec[i], OPIHI_FLT, NELEM);
+    }
   }
   
@@ -128,8 +157,11 @@
   }
 
-  Nstart = 0;
-  N = 0;
-  done = FALSE;
-  while (!done) {
+  // we have a working buffer read from the file. we parse the lines in the working buffer
+  // until we reach the last chunk without an EOL char.  at that point, we shift the start
+  // of the last (partial) line to the start of the buffer and re-fill.
+
+  Nstart = 0; // location of the last valid byte in the buffer (start filling here)
+  Nelem = 0; // number of valid rows read (vector elements)
+  while (TRUE) {
     Nbytes = 0x10000 - Nstart;
     bzero (&buffer[Nstart], Nbytes);
@@ -139,61 +171,83 @@
       break;
     }
-    if (nread == 0) break;
-    nbytes = nread + Nstart;
+    if (nread == 0) break; // end of the file
+    // nbytes = nread + Nstart;
     
-    status = TRUE;
-    c0 = buffer; 
-    while (status) {
-      c1 = strchr (c0, '\n');
+    int bufferStatus = TRUE; 
+    c0 = buffer; // c0 always marks the start of a line
+    while (bufferStatus) {
+      c1 = strchr (c0, '\n'); // find the end of this current line
       if (c1 == (char *) NULL) {
 	Nstart = strlen (c0);
 	memmove (buffer, c0, Nstart);
-	status = FALSE;
-      } else {
-	*c1 = 0;
-      }      
-      if ((*c0 != '#') && (*c0 != '!')) {
-	for (i = 0; (i < Nvec) && status; i++) {
+	bufferStatus = FALSE;
+	continue;
+      }
+      *c1 = 0; // mark the end of the line 
+
+      if (*c0 == '#') {	c0 = c1 + 1; continue; }
+      if (*c0 == '!') {	c0 = c1 + 1; continue; }
+
+      // parse the vectors in this line.  this code is a bit inefficient: each column
+      // requires a separate pass through the line.
+
+      int lineStatus = TRUE;
+      for (i = 0; i < Nvec; i++) {
+	int ivalue;
+	double dvalue;
+	time_t tvalue;
+	int readStatus;
+	// need to make the if cases for coltype[i]
+	switch (coltype[i]) {
+	  case COLTYPE_INT:
+	    readStatus = IsCSV ? iparse_csv (&ivalue, col[i], c0) : iparse (&ivalue, col[i], c0);
+	    vec[i][0].elements.Int[Nelem] = readStatus ? ivalue : 0;
+	    break;
+	  case COLTYPE_FLT:
+	    readStatus = IsCSV ? dparse_csv (&dvalue, col[i], c0) : dparse (&dvalue, col[i], c0);
+	    vec[i][0].elements.Flt[Nelem] = readStatus ? dvalue : NAN;
+	    break;
+	  case COLTYPE_TIME:
+	    readStatus = IsCSV ? tparse_csv (&tvalue, col[i], c0) : tparse (&tvalue, col[i], c0);
+	    dvalue = TimeValue (tvalue, TimeReference, TimeFormat);
+	    vec[i][0].elements.Flt[Nelem] = readStatus ? dvalue : NAN;
+	    break;
+	}
+	if (!readStatus && VERBOSE) {
 	  if (IsCSV) {
-	    status = dparse_csv (&value, col[i], c0);
+	    gprint (GP_ERR, "suspect field: %d (%s) in %s\n", col[i], argv[2*i+2], c0);
 	  } else {
-	    status = dparse (&value, col[i], c0);
+	    gprint (GP_ERR, "suspect field: %d in %s\n", col[i], c0);
 	  }
-	  if (status) {
-	      vec[i][0].elements.Flt[N] = value;
-	  } else {
-	      vec[i][0].elements.Flt[N] = NAN;
-	      if (VERBOSE) {
-		  if (IsCSV) {
-		      gprint (GP_ERR, "suspect field: %d (%s) in %s\n", col[i], argv[2*i+2], c0);
-		  } else {
-		      gprint (GP_ERR, "suspect field: %d in %s\n", col[i], c0);
-		  }
-	      }
-	  }
-	}
-	if (status) {
-	    N++;
-	} else {
-	    if (VERBOSE && FALSE) {
-		char temp[32];
-		strncpy (temp, c0, 32);
-		temp[31] = 0;
-		gprint (GP_ERR, "skip line %s\n\n", temp);
-	    }
-	}
-      }
-      c0 = c1 + 1;
-      if (N == NELEM) {
+	}
+	lineStatus &= readStatus;
+      }
+      if (!lineStatus && VERBOSE) {
+	char temp[32];
+	strncpy (temp, c0, 32);
+	temp[31] = 0;
+	gprint (GP_ERR, "skip line %s\n\n", temp);
+      }
+      Nelem ++;
+      if (Nelem == NELEM) {
 	NELEM += 1000;
 	for (i = 0; i < Nvec; i++) {
-	  REALLOCATE (vec[i][0].elements.Flt, opihi_flt, NELEM);
-	}
-      }
+	  if (coltype[i] == COLTYPE_INT) {
+	    REALLOCATE (vec[i][0].elements.Int, opihi_int, NELEM);
+	  } else {
+	    REALLOCATE (vec[i][0].elements.Flt, opihi_flt, NELEM);
+	  }
+	}
+      }
+      c0 = c1 + 1;
     }
   }
   for (i = 0; i < Nvec; i++) {
-    REALLOCATE (vec[i][0].elements.Flt, opihi_flt, MAX (N,1));
-    vec[i][0].Nelements = N;
+    if (coltype[i] == COLTYPE_INT) {
+      REALLOCATE (vec[i][0].elements.Int, opihi_int, MAX (Nelem,1));
+    } else {
+      REALLOCATE (vec[i][0].elements.Flt, opihi_flt, MAX (Nelem,1));
+    }
+    vec[i][0].Nelements = Nelem;
   }
   
@@ -399,104 +453,2 @@
   return (TRUE);
 }
-
-# if (0)
-int read_vectors_csv (int argc, char **argv, int Nskip, FILE *f) {
-  
-  int i, j, Nvec, *col, done, status;
-  int Nbytes, nbytes, Nstart, NELEM, N, nread;
-  char *colstr, *c0, *c1, *buffer, *extname;
-  double value;
-  Vector **vec;
-
-  Nvec = (argc - 1) / 2;
-  ALLOCATE (vec, Vector *, Nvec);
-  ALLOCATE (col, int, Nvec);
-
-  for (i = 0; i < Nvec; i++) {
-    if ((vec[i] = SelectVector (argv[2*i + 1], ANYVECTOR, TRUE)) == NULL) {
-      gprint (GP_ERR, "USAGE: read name N name N ...\n");
-      free (vec);
-      free (col);
-      return (FALSE);    
-    }
-    // XXX we could allow flags (eg, N.i) to specify INT vs FLT types vectors...
-    colstr = argv[2*i+2];
-    for (j = 0; j < strlen (colstr); j++) {
-      if (!isdigit(colstr[j])) {
-	gprint (GP_ERR, "USAGE: read name N name N ...\n");
-	free (vec);
-	free (col);
-	return (FALSE);    
-      }
-    }
-    col[i] = atof (colstr);
-  }
-
-  // currently, all read vectors are forced to be type FLT
-  NELEM = 1000;
-  for (i = 0; i < Nvec; i++) {
-    ResetVector (vec[i], OPIHI_FLT, NELEM);
-  }
-  
-  ALLOCATE (buffer, char, 0x10001);
-  bzero (buffer, 0x10001);
-  for (i = 0; i < Nskip; i++) {
-    scan_line (f, buffer);
-  }
-
-  Nstart = 0;
-  N = 0;
-  done = FALSE;
-  while (!done) {
-    Nbytes = 0x10000 - Nstart;
-    bzero (&buffer[Nstart], Nbytes);
-    nread = fread (&buffer[Nstart], 1, Nbytes, f);
-    if (ferror (f)) {
-      perror ("error reading data file");
-      break;
-    }
-    if (nread == 0) break;
-    nbytes = nread + Nstart;
-    
-    status = TRUE;
-    c0 = buffer; 
-    while (status) {
-      c1 = strchr (c0, '\n');
-      if (c1 == (char *) NULL) {
-	Nstart = strlen (c0);
-	memmove (buffer, c0, Nstart);
-	status = FALSE;
-      } else {
-	*c1 = 0;
-      }      
-      if ((*c0 != '#') && (*c0 != '!')) {
-	for (i = 0; (i < Nvec) && status; i++) {
-	  status = dparse_csv (&value, col[i], c0);
-	  vec[i][0].elements.Flt[N] = value;
-	  if (!status) vec[i][0].elements.Flt[N] = NAN;
-	}
-	if (status) N++;
-      }
-      c0 = c1 + 1;
-      if (N == NELEM) {
-	NELEM += 1000;
-	for (i = 0; i < Nvec; i++) {
-	  REALLOCATE (vec[i][0].elements.Flt, opihi_flt, NELEM);
-	}
-      }
-	
-    }
-  }
-  for (i = 0; i < Nvec; i++) {
-    REALLOCATE (vec[i][0].elements.Flt, opihi_flt, MAX (N,1));
-    vec[i][0].Nelements = N;
-  }
-  
-  free (vec);
-  free (col);
-  free (buffer);
-  return (TRUE);
-
-}
-
-# endif
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.sh
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.sh	(revision 33115)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.sh	(revision 33115)
@@ -0,0 +1,145 @@
+
+list tests
+ #test1
+ #test2
+ #test3
+ #test4
+ test5
+ test6
+end
+
+# basic read
+macro test1
+ 
+ $PASS = 1
+
+ data read.t1.dat
+ read t1 1 t2 2 t3 3
+
+ if (t1[0] != 1) set PASS = 0
+ if (t1[1] != 2) set PASS = 0
+ if (t1[2] != 4) set PASS = 0
+
+ if (t2[0] != 5) set PASS = 0
+ if (t2[1] != 3) set PASS = 0
+ if (t2[2] != 5) set PASS = 0
+
+ if (t3[0] != 8) set PASS = 0
+ if (t3[1] != 9) set PASS = 0
+ if (t3[2] != 7) set PASS = 0
+end
+
+# basic read with int types
+macro test2
+ 
+ $PASS = 1
+
+ data read.t1.dat
+ read t1:int 1 t2:int 2 t3:int 3
+
+ # note: no way to verify the mode of a vector
+ vectors
+
+ if (t1[0] != 1) set PASS = 0
+ if (t1[1] != 2) set PASS = 0
+ if (t1[2] != 4) set PASS = 0
+
+ if (t2[0] != 5) set PASS = 0
+ if (t2[1] != 3) set PASS = 0
+ if (t2[2] != 5) set PASS = 0
+
+ if (t3[0] != 8) set PASS = 0
+ if (t3[1] != 9) set PASS = 0
+ if (t3[2] != 7) set PASS = 0
+end
+
+# basic read with comments
+macro test3
+ 
+ $PASS = 1
+
+ data read.t2.dat
+ read t1 1 t2 2 t3 3
+
+ if (t1[0] != 1) set PASS = 0
+ if (t1[1] != 2) set PASS = 0
+ if (t1[2] != 4) set PASS = 0
+
+ if (t2[0] != 5) set PASS = 0
+ if (t2[1] != 3) set PASS = 0
+ if (t2[2] != 5) set PASS = 0
+
+ if (t3[0] != 8) set PASS = 0
+ if (t3[1] != 9) set PASS = 0
+ if (t3[2] != 7) set PASS = 0
+end
+
+# basic read with comments & int types
+macro test4
+ 
+ $PASS = 1
+
+ data read.t2.dat
+ read t1:int 1 t2:int 2 t3:int 3
+
+ # note: no way to verify the mode of a vector
+ vectors
+
+ if (t1[0] != 1) set PASS = 0
+ if (t1[1] != 2) set PASS = 0
+ if (t1[2] != 4) set PASS = 0
+
+ if (t2[0] != 5) set PASS = 0
+ if (t2[1] != 3) set PASS = 0
+ if (t2[2] != 5) set PASS = 0
+
+ if (t3[0] != 8) set PASS = 0
+ if (t3[1] != 9) set PASS = 0
+ if (t3[2] != 7) set PASS = 0
+end
+
+# read with invalid entries
+macro test5
+ 
+ $PASS = 1
+
+ data read.t3.dat
+ read t1 1 t2 2 t3 3
+
+ if (t1[0] != 1) set PASS = 0
+ if (t1[1] != 2) set PASS = 0
+ if (t1[2] != 4) set PASS = 0
+
+ if (t2[0] != 5) set PASS = 0
+ if (not(isnan(t2[1]))) set PASS = 0
+ if (t2[2] != 5) set PASS = 0
+
+ if (t3[0] != 8) set PASS = 0
+ if (t3[1] != 9) set PASS = 0
+ if (not(isnan(t3[2]))) set PASS = 0
+end
+
+# read with invalid entries & int types
+macro test6
+ 
+ $PASS = 1
+
+ data read.t3.dat
+ read t1:int 1 t2:int 2 t3:int 3
+
+ # note: no way to verify the mode of a vector
+ vectors
+
+ if (t1[0] != 1) set PASS = 0
+ if (t1[1] != 2) set PASS = 0
+ if (t1[2] != 4) set PASS = 0
+
+ if (t2[0] != 5) set PASS = 0
+ if (t2[1] != 0) set PASS = 0
+ if (t2[2] != 5) set PASS = 0
+
+ if (t3[0] != 8) set PASS = 0
+ if (t3[1] != 9) set PASS = 0
+ if (t3[2] != 0) set PASS = 0
+end
+
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t1.dat
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t1.dat	(revision 33115)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t1.dat	(revision 33115)
@@ -0,0 +1,3 @@
+1 5 8
+2 3 9
+4 5 7
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t2.dat
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t2.dat	(revision 33115)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t2.dat	(revision 33115)
@@ -0,0 +1,5 @@
+# comment line
+1 5 8
+2 3 9
+! comment line 2
+4 5 7
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t3.dat
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t3.dat	(revision 33115)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t3.dat	(revision 33115)
@@ -0,0 +1,3 @@
+1 5 8
+2 X 9
+4 5 Y
Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t4.dat
===================================================================
--- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t4.dat	(revision 33115)
+++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/test/read.t4.dat	(revision 33115)
@@ -0,0 +1,7 @@
+1 5 8 2000/01/01,00:00:00
+2 3 9 2000/01/01,01:00:00
+4 5 7 2000/01/01,02:00:00
+4 5 7 now
+4 5 7 today
+4 5 7 2451544.750000j
+4 5 7 51544.500000J