Changeset 41341 for trunk/Ohana/src/opihi/lib.shell/VectorIO.c

Timestamp:

Apr 16, 2020, 2:04:27 PM (6 years ago)

Author:

tdeboer

Message:

quick undo

File:

• trunk/Ohana/src/opihi/lib.shell/VectorIO.c (modified) (10 diffs)

trunk/Ohana/src/opihi/lib.shell/VectorIO.c

@@ -2 +2 @@
 void gfits_compress_timing ();
   
+static int VectorGetMaxStringLength (Vector *vec) {
+
+  int MaxLength = 0;
+
+  if (vec[0].type != OPIHI_STR) return 0;
+
+  for (int i = 0; i < vec[0].Nelements; i++) {
+    MaxLength = MAX (MaxLength, strlen(vec[0].elements.Str[i]));
+  }
+  return MaxLength;
+}
+
 // write a set of vectors to a FITS FTable structure (vectors names become fits column names)
 static int WriteVectorTable (FTable *ftable, char *extname, Vector **vec, int Nvec, char *format, char nativeOrder) {
@@ -7 +19 @@
   int j;
 
-  char *tformat = NULL;
-
   Header *theader = ftable->header;
   gfits_create_table_header (theader, "BINTABLE", extname);
 
-  ALLOCATE (tformat, char, 2*Nvec);
+  // allocate an array of strings to represent the format for each output field
+  // formats include single column formats (BIJKDE) and multi-column formats (e.g., 2I)
+  // we will have no more than Nvec fields (but we can have fewer)
+  int Nfield = 0;
+  ALLOCATE_PTR (tformat, char *, Nvec);
+  ALLOCATE_PTR (Nelement, int, Nvec);
+
   if (format) {
-    // the bintable format string can only define the byte-width of each field.  valid output columns are currently:
+    // the bintable format string can defines the byte-width of each field and number of elements (columns per field).
+    // valid output columns are currently:
     // B (char), I (16 bit short), J (32 bit int), E (32 bit float), D (64 bit double).
-    // the format string is just the sequence of types, eg: LIIJEED
-    // validate the format string
+    // the format string is just the sequence of types, eg: LIIJEED.
+    // it may have spaces or integer element counts:
+    // "2D 4I EEJ"
+
+    // *** validate the format string
+
+    // as I parse each elements, if it is a digit, I need parse that value
+
     char *ptr = format;
-    for (j = 0; j < Nvec; j++) {
+    for (j = 0; j < Nvec; ) {
       while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
       if (*ptr == 0) {
@@ -25 +48 @@
         goto escape;
       }
-      if ((*ptr != 'B') && (*ptr != 'I') && (*ptr != 'J') && (*ptr != 'K') && (*ptr != 'D') && (*ptr != 'E')) {
+
+      // is there a leading integer?
+      char *endptr;
+      Nelement[Nfield] = strtol (ptr, &endptr, 10);
+      if (endptr == ptr) {
+        Nelement[Nfield] = 1;
+      }
+      ptr = endptr; // this should now point at the letter that is the format type
+      if ((*ptr != 'B') && (*ptr != 'I') && (*ptr != 'J') && (*ptr != 'K') && (*ptr != 'D') && (*ptr != 'E') && (*ptr != 'A')) {
         gprint (GP_ERR, "error in binary table format %s: invalid character %c\n", format, *ptr);
         goto escape;
       }
-      tformat[2*j + 0] = *ptr;
-      tformat[2*j + 1] = 0; // a bit sleazy : use a 2xN string to store N 1-byte strings
+
+      int Nchar = snprintf (tformat[Nfield], 0, "%d%c", Nelement[Nfield], *ptr);
+      ALLOCATE (tformat[Nfield], char, Nchar + 1);
+      int Nout = snprintf (tformat[Nfield], Nchar + 1, "%d%c", Nelement[Nfield], *ptr);
+      myAssert (Nout <= Nchar, "oops");
+
+      // tformat[2*j + 0] = *ptr;
+      // tformat[2*j + 1] = 0; // a bit sleazy : use a 2xN string to store N 1-byte strings
+
+      // For numeric multi-valued fields, the number of elements from the format defines the number of
+      // vectors which go into that field.  For string-type vectors, the format specifies the maximum number of characters
+      // that go into the field.
+
+      // for example, a format code of 3E should match a list of three numeric-type vectors while a format code of 15A should
+      // match a single string-type vector which will supply up to 15 chars per row.
+
+      if (*ptr != 'A') {
+        j += Nelement[Nfield]; // advance past Nelement vectors
+      } else {
+        if (vec[j][0].type != OPIHI_STR) {
+          gprint (GP_ERR, "error in binary table format %s (mismatch between string format and numeric vector %s)\n", format, vec[j][0].name);
+          goto escape;
+        }
+        j ++; // advance past a single string-type vector (validate that?)
+      }
+      if (j > Nvec) {
+        gprint (GP_ERR, "error in binary table format %s (too few vectors for listed field)\n", format);
+        goto escape;
+      }
+
       ptr ++;
+      Nfield ++;
     }
     while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
@@ -40 +100 @@
   } else {
     for (j = 0; j < Nvec; j++) {
-      // if the format is not defined, just use the native byte-widths
-      tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'K'; // this depends on opihi_int == int64_t for Int
-      tformat[2*j + 1] = 0;
-    }
+      switch (vec[j][0].type) {
+        case OPIHI_FLT:
+        case OPIHI_INT:
+          // if the format is not defined, just use the native byte-widths
+          ALLOCATE (tformat[j], char, 2);
+          tformat[j][0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'K';
+          tformat[j][1] = 0;
+          Nelement[j] = 1;
+          break;
+        case OPIHI_STR:
+          // we need to examine the vector to determine the length
+          Nelement[j] = VectorGetMaxStringLength(vec[j]); 
+          int Nchar = snprintf (tformat[j], 0, "%d%c", Nelement[j], 'A');
+          ALLOCATE (tformat[j], char, Nchar + 1);
+          int Nout = snprintf (tformat[j], Nchar + 1, "%d%c", Nelement[j], 'A');
+          myAssert (Nout <= Nchar, "oops");
+          break;
+      }
+    }
+    Nfield = Nvec;
   }
 
@@ -49 +125 @@
   // output table (because the data goes to the named column below).  need to enforce
   // this somehow
-  for (j = 0; j < Nvec; j++) {
-    gfits_define_bintable_column (theader, &tformat[2*j], vec[j][0].name, NULL, NULL, 1.0, 0.0);
+  int ivec = 0;
+  for (j = 0; j < Nfield; j++) {
+    // XX need to loop over fields, and skip the additional vectors that are part of a field
+    // this call supported multiple columns per named field
+    gfits_define_bintable_column (theader, tformat[j], vec[ivec][0].name, NULL, NULL, 1.0, 0.0);
+    if (vec[ivec][0].type == OPIHI_STR) {
+      ivec ++;
+    } else {
+      ivec += Nelement[j];
+    }
+  }
+
+  // need to free the array 
+  for (j = 0; j < Nfield; j++) {
+    free (tformat[j]);
   }
   free (tformat);
@@ -57 +146 @@
   gfits_create_table (theader, ftable);
 
+  // I need to add each vector in order, but I need to
+  // track which field it corresponds to.
+
   // add the vectors to the output array
-  for (j = 0; j < Nvec; j++) {
-    if (vec[j][0].type == OPIHI_FLT) {
-      gfits_set_bintable_column_reformat (theader, ftable, vec[j][0].name, "double",  vec[j][0].elements.Flt, vec[j][0].Nelements, nativeOrder);
+  for (ivec = 0, j = 0; j < Nfield; j++) {
+    // the first vector provides the name for the field
+    char *fieldname = vec[ivec][0].name;
+
+    if (vec[ivec][0].type == OPIHI_STR) {
+      // string-type vectors need to be copied into a contiguous buffer with the right dimensions:
+      Vector *thisvec = vec[ivec];
+
+      ALLOCATE_PTR (strbuffer, char, Nelement[j]*thisvec->Nelements);
+      for (int i = 0; i < thisvec->Nelements; i++) {
+        int nChar = MIN (strlen (thisvec->elements.Str[i]), Nelement[j]);
+        // fprintf (stderr, "%d %d %d : %d : %d : %s\n", ivec, j, i, Nelement[j], nChar, thisvec->elements.Str[i]);
+        memcpy (&strbuffer[i*Nelement[j]], thisvec->elements.Str[i], nChar);
+      }
+      gfits_set_bintable_column (theader, ftable, fieldname, strbuffer, thisvec->Nelements);
+      free (strbuffer);
+      ivec ++;
     } else {
-//    gfits_set_bintable_column_reformat (theader, ftable, vec[j][0].name, "int",     vec[j][0].elements.Int, vec[j][0].Nelements, nativeOrder);
-      gfits_set_bintable_column_reformat (theader, ftable, vec[j][0].name, "int64_t", vec[j][0].elements.Int, vec[j][0].Nelements, nativeOrder);
+      for (int k = 0; k < Nelement[j]; k++, ivec++) {
+        Vector *thisvec = vec[ivec];
+        switch (thisvec->type) {
+          case OPIHI_FLT:
+            gfits_set_bintable_column_reformat (theader, ftable, fieldname, "double",  thisvec->elements.Flt, thisvec->Nelements, k, nativeOrder);
+            break;
+          case OPIHI_INT:
+            gfits_set_bintable_column_reformat (theader, ftable, fieldname, "int64_t", thisvec->elements.Int, thisvec->Nelements, k, nativeOrder);
+            break;
+        }
+      }
     }
   }
@@ -110 +225 @@
 }
 
+// insert a new header line, return end pointer
+char *gfits_insert_header_line (Header *header, char *endptr, char *newline) {
+
+  if (0) {
+    char temp1[32], temp2[32];
+    memset (temp1, 0, 32);
+    memset (temp2, 0, 32);
+
+    if (endptr) {
+      memcpy (temp1, endptr, 8);
+    }
+    memcpy (temp2, newline, 8);
+    fprintf (stderr, "insert: %s : %s\n", temp1, temp2);
+  }
+
+  /* find the END of the header, if not supplied */
+  if (!endptr) {
+    endptr = gfits_header_field (header, "END", 1);
+    if (endptr == NULL) return NULL; 
+  }
+
+  /* is there enough space for 1 more line? */
+  if (header[0].datasize - (endptr - (header[0].buffer)) < 2*FT_LINE_LENGTH) {
+    // no, so expand by a full header block (FT_RECORD_SIZE = 32*80 = 2880)
+    header[0].datasize += FT_RECORD_SIZE;
+    REALLOCATE (header[0].buffer, char, header[0].datasize);
+    // re-find the "END" marker, in case new memory block is used 
+    endptr = gfits_header_field (header, "END", 1);
+    if (endptr == NULL) return NULL; 
+    memset (endptr + FT_LINE_LENGTH, ' ', FT_RECORD_SIZE);
+  }
+
+  /* push END line back 1 */
+  memmove ((endptr + FT_LINE_LENGTH), endptr, FT_LINE_LENGTH);
+  memset (endptr, ' ', FT_LINE_LENGTH);
+
+  strncpy (endptr, newline, 80);
+  endptr += FT_LINE_LENGTH;
+
+  return endptr;
+}
+  
+static char *rawkeywords[] = {"SIMPLE", "BITPIX", "NAXIS", "PCOUNT", "GCOUNT", "EXTEND", "EXTNAME", "BSCALE", "BZERO", "TFIELDS", "TFORM", "TZERO", "TSCAL", "        ", NULL};
+
 // write a set of vectors to a FITS file (vectors names become fits column names)
-int WriteVectorTableFITS (char *filename, char *extname, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile) {
+int WriteVectorTableFITS (char *filename, char *extname, Header *extraheader, Vector **vec, int Nvec, int append, char *compress, char *format, int Ntile) {
   
   Header rawheader;
@@ -139 +298 @@
   rawtable.header = &rawheader;
   if (!WriteVectorTable (&rawtable, extname, vec, Nvec, format, (compress != NULL))) goto escape;
-  // NOTE : for compression, the table is constructed in native by order 
+  // NOTE : for compression, the table is constructed in native byte order 
 
   FTable *outtable = &rawtable;
@@ -163 +322 @@
 
   if (!append) {
+    // generate a blank PHU header
     Header header;
     Matrix matrix;
@@ -176 +336 @@
   }
 
+  if (extraheader) {
+    // copy keywords which are not the standard or table keywords
+    char *buf = extraheader->buffer;
+    char *endptr = NULL;
+    for (int i = 0; i < extraheader->datasize; i+= FT_LINE_LENGTH, buf += FT_LINE_LENGTH) {
+
+      if (0) { 
+        char temp1[32];
+        memset (temp1, 0, 32);
+        memcpy (temp1, buf, 8);
+        fprintf (stderr, "buffer: %s\n", temp1);
+      }
+
+      for (int j = 0; rawkeywords[j] != NULL; j++) {
+        if (!strncmp (buf, rawkeywords[j], strlen(rawkeywords[j]))) goto skip_insert;
+      }
+      endptr = gfits_insert_header_line (outheader, endptr, buf);
+      if (!endptr) {
+        gprint (GP_ERR, "failed to update FITS header with extra keywords\n");
+        return (FALSE);
+      }
+    skip_insert:
+      continue;
+    }
+  }
+
   // write the actual table data
   gfits_fwrite_Theader (f, outheader);