Index: /branches/eam_branches/ipp-20111122/Ohana/src/libfits/include/gfitsio.h =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/libfits/include/gfitsio.h (revision 33308) +++ /branches/eam_branches/ipp-20111122/Ohana/src/libfits/include/gfitsio.h (revision 33309) @@ -184,4 +184,5 @@ int gfits_get_bintable_column PROTO((Header *header, FTable *table, char *label, void **data)); int gfits_get_bintable_column_type PROTO((Header *header, char *label, char *type, int *Nval)); +int gfits_get_bintable_column_type_by_N PROTO((Header *header, int N, char *type, int *Nval)); void *gfits_get_bintable_column_data PROTO((Header *header, FTable *table, char *label, char *type, off_t *Nrow, int *Ncol)); int gfits_get_table_column PROTO((Header *header, FTable *table, char *label, void **data)); Index: /branches/eam_branches/ipp-20111122/Ohana/src/libfits/table/F_get_column.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/libfits/table/F_get_column.c (revision 33308) +++ /branches/eam_branches/ipp-20111122/Ohana/src/libfits/table/F_get_column.c (revision 33309) @@ -126,8 +126,22 @@ /***********************/ +int gfits_get_bintable_column_type_by_N (Header *header, int N, char *type, int *Nval) { + + int Nbytes; + char field[80], format[80]; + + assert (N > 0); + + sprintf (field, "TFORM%d", N); + if (!gfits_scan (header, field, "%s", 1, format)) return FALSE; + if (!gfits_bintable_format (format, type, Nval, &Nbytes)) return (FALSE); + return (TRUE); +} + +/***********************/ int gfits_get_bintable_column_type (Header *header, char *label, char *type, int *Nval) { - int i, N, Nfields, Nbytes; - char tlabel[80], field[80], format[80]; + int i, N, Nfields; + char tlabel[80], field[80]; if (label == (char *) NULL) return (FALSE); @@ -144,8 +158,5 @@ N = i - 1; - sprintf (field, "TFORM%d", N); - gfits_scan (header, field, "%s", 1, format); - - if (!gfits_bintable_format (format, type, Nval, &Nbytes)) return (FALSE); + if (!gfits_get_bintable_column_type_by_N (header, N, type, Nval)) return FALSE; return (TRUE); } Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/read_vectors.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/read_vectors.c (revision 33308) +++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/read_vectors.c (revision 33309) @@ -297,4 +297,12 @@ } + // XXX ReadAll needs: deal with Extnum vs Extname, save vectors, etc + // ReadAll = FALSE; + // if ((N = get_argument (argc, argv, "-all"))) { + // remove_argument (N, &argc, argv); + // ReadAll = atoi (extname); + // if (argc != 1) ESCAPE ("-all option cannot be mixed with selected field"); + // } + if (argc < 2) ESCAPE ("USAGE: read -fits extension [-extnum] [-keyword key] name name ..."); Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/mextract.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/mextract.c (revision 33308) +++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/mextract.c (revision 33309) @@ -5,4 +5,7 @@ # define PARALLEL_SERIAL 0 # define MAX_PATH_LENGTH 1024 + +int HostTableLaunchJobs (HostTable *table, char *basecmd); +Vector **MergeVectors (Vector **vec, int *Nvec, Vector **invec, int Ninvec); int mextract (int argc, char **argv) { @@ -93,5 +96,24 @@ // load fields from file - // ReadVectorTables + int Nvec = 0; + Vector **vec = NULL; + for (i = 0; i < table->Nhosts; i++) { + char resultFile[MAX_PATH_LENGTH]; + snprintf (resultFile, MAX_PATH_LENGTH, "%s/dvo.results.fits", table->hosts[i].pathname); + + int Ninvec = 0; + Vector **invec = ReadVectorTableFITS (resultFile, "MEXTRACT", &Ninvec); + + vec = MergeVectors (vec, &Nvec, invec, Ninvec); + if (vec != invec) { + FreeVectorArray (invec, Ninvec); + } + } + + for (i = 0; i < Nvec; i++) { + AssignVector (vec[i], vec[i]->name, ANYVECTOR, TRUE); + } + free (vec); + return TRUE; } @@ -449,2 +471,34 @@ } +// take two arrays of vectors and merge equal named vectors. +// for ease, require that the order of the names match & number of vectors match +Vector **MergeVectors (Vector **vec, int *Nvec, Vector **invec, int Ninvec) { + + int i, j; + + if (vec == NULL) { + *Nvec = Ninvec; + return invec; + } + + myAssert (*Nvec == Ninvec, "programming error (1)"); + + for (i = 0; i < Ninvec; i++) { + myAssert (!strcmp(vec[i]->name, invec[i]->name), "programming error (2)"); + myAssert (vec[i]->type == invec[i]->type, "programming error (2)"); + + if (vec[i]->type == OPIHI_FLT) { + REALLOCATE (vec[i]->elements.Flt, opihi_flt, vec[i]->Nelements + invec[i]->Nelements); + for (j = 0; j < invec[i]->Nelements; j++) { + vec[i]->elements.Flt[i+j] = vec[i]->elements.Flt[j]; + } + } else { + REALLOCATE (vec[i]->elements.Int, opihi_int, vec[i]->Nelements + invec[i]->Nelements); + for (j = 0; j < invec[i]->Nelements; j++) { + vec[i]->elements.Int[i+j] = vec[i]->elements.Int[j]; + } + } + vec[i]->Nelements += invec[i]->Nelements; + } + return vec; +} Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/region_list.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/region_list.c (revision 33308) +++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/region_list.c (revision 33309) @@ -113,5 +113,5 @@ /* check for Region selection from display */ if ((N = get_argument (*argc, argv, "-skyregion"))) { - if (N + 5 >= *argc) { + if (N + 4 >= *argc) { gprint (GP_ERR, "USAGE: -skyregion (RA) (RA) (DEC) (DEC)\n"); FreeSkyRegionSelection (selection); Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvomath.h =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvomath.h (revision 33308) +++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvomath.h (revision 33309) @@ -111,6 +111,8 @@ /* vector handling */ +void InitVectors PROTO((void)); +void FreeVectors PROTO((void)); Vector *InitVector PROTO((void)); -void InitVectors PROTO((void)); +void FreeVectorArray PROTO((Vector **vec, int Nvec)); int CopyVector PROTO((Vector *out, Vector *in)); int ResetVector PROTO((Vector *vec, char type, int Nelements)); @@ -127,5 +129,9 @@ int ListVectors PROTO((void)); Vector *SelectVector PROTO((char *name, int mode, int verbose)); +int AssignVector PROTO((Vector *vec, char *name, int mode, int verbose)); + +/* vector IO functions */ int WriteVectorTableFITS PROTO((char *filename, char *extname, Vector **vec, int Nvec, int append, char *format)); +Vector **ReadVectorTableFITS PROTO((char *filename, char *extname, int *Nvec)); /* buffer handling */ Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/Makefile =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/Makefile (revision 33308) +++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/Makefile (revision 33309) @@ -24,4 +24,5 @@ $(SDIR)/VariableOps.$(ARCH).o \ $(SDIR)/VectorOps.$(ARCH).o \ +$(SDIR)/VectorIO.$(ARCH).o \ $(SDIR)/check_stack.$(ARCH).o \ $(SDIR)/command.$(ARCH).o \ Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorIO.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorIO.c (revision 33309) +++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorIO.c (revision 33309) @@ -0,0 +1,234 @@ +# include "opihi.h" + +// write a set of vectors to a FITS file (vectors names become fits column names) +int WriteVectorTableFITS (char *filename, char *extname, Vector **vec, int Nvec, int append, char *format) { + + char *tformat = NULL; + Header header; + Matrix matrix; + Header theader; + FTable ftable; + + int j; + FILE *f = NULL; + + /* open file for outuput */ + if (append) { + f = fopen (filename, "a"); + } else { + f = fopen (filename, "w"); + } + if (f == (FILE *) NULL) { + gprint (GP_ERR, "can't open file for write\n"); + return (FALSE); + } + + if (!append) { + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + } + + gfits_create_table_header (&theader, "BINTABLE", extname); + + ALLOCATE (tformat, char, 2*Nvec); + if (format) { + // the bintable format string can only define the byte-width of each field. valid output columns are currently: + // B (char), I (16 bit short), J (32 bit int), E (32 bit float), D (64 bit double). + // the format string is just the sequence of types, eg: LIIJEED + // validate the format string + char *ptr = format; + for (j = 0; j < Nvec; j++) { + while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++; + if (*ptr == 0) { + gprint (GP_ERR, "error in binary table format %s (insufficient format chars)\n", format); + goto escape; + } + if ((*ptr != 'B') && (*ptr != 'I') && (*ptr != 'J') && (*ptr != 'D') && (*ptr != 'E')) { + gprint (GP_ERR, "error in binary table format %s: invalid character %c\n", format, *ptr); + goto escape; + } + tformat[2*j + 0] = *ptr; + tformat[2*j + 1] = 0; // a bit sleazy : use a 2xN string to store N 1-byte strings + ptr ++; + } + while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++; + if (*ptr) { + gprint (GP_ERR, "error in binary table format %s (extra characters in format)\n", format); + goto escape; + } + } else { + for (j = 0; j < Nvec; j++) { + // if the format is not defined, just use the native byte-widths + tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'J'; + tformat[2*j + 1] = 0; + } + } + + // define the columns of the table. XXX NOTE: we cannot have duplicate names in + // output table (because the data goes to the named column below). need to enforce + // this somehow + for (j = 0; j < Nvec; j++) { + gfits_define_bintable_column (&theader, &tformat[2*j], vec[j][0].name, NULL, NULL, 1.0, 0.0); + } + free (tformat); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + // add the vectors to the output array + for (j = 0; j < Nvec; j++) { + if (vec[j][0].type == OPIHI_FLT) { + gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "double", vec[j][0].elements.Flt, vec[j][0].Nelements); + } else { + gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "int", vec[j][0].elements.Int, vec[j][0].Nelements); + } + } + + if (!append) { + gfits_fwrite_header (f, &header); + gfits_fwrite_matrix (f, &matrix); + gfits_free_header (&header); + gfits_free_matrix (&matrix); + } + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + + gfits_free_header (&theader); + gfits_free_table (&ftable); + + fclose (f); + fflush (f); + return (TRUE); + + escape: + if (!append) { + gfits_free_header (&header); + gfits_free_matrix (&matrix); + } + gfits_free_header (&theader); + gfits_free_table (&ftable); + + if (tformat) free (tformat); + fclose (f); + fflush (f); + return (FALSE); +} + +// read the complete set of vectors from the given FITS file & extension +// XXX not quite right : I need to merge multiple vectors together: +// do not associate with an Opihi vector until later in mextract +Vector **ReadVectorTableFITS (char *filename, char *extname, int *nvec) { + + Header header; + Header theader; + FTable ftable; + + int i, j, k; + FILE *f = NULL; + + /* open file for input */ + f = fopen (filename, "r"); + if (f == (FILE *) NULL) { + gprint (GP_ERR, "can't open file for read\n"); + return NULL; + } + + ftable.header = &header; + + // read the full table data into a buffer + if (!gfits_fread_ftable (f, &ftable, extname)) { + fclose (f); + return NULL; + } + + // XXX handle binary and ascii tables (see read_vectors.c) + // find the columns in the table + int Nfields; + gfits_scan (&header, "TFIELDS", "%d", 1, &Nfields); + int Nrows = header.Naxis[1]; + + // how many output vectors do we need? depends on number of columns per field, but min is Nfields + int Nvec = 0; + int NVEC = Nfields; + Vector **vec = NULL; + ALLOCATE (vec, Vector *, NVEC); + + for (i = 0; i < Nfields; i++) { + int Nval; + char type[16], label[16], name[80]; + + // determine the column name, data type, and number of sub-fields + sprintf (label, "TTYPE%d", i + 1); + int status = gfits_scan (&header, label, "%s", 1, name); + assert (status); + + status = gfits_get_bintable_column_type_by_N (&header, i + 1, type, &Nval); + assert (status); + + if (Nvec + Nval >= NVEC) { + NVEC = Nvec + Nval + 16; + REALLOCATE (vec, Vector *, NVEC); + } + + int vecType = OPIHI_INT; + if (!strcmp (type, "double") || !strcmp (type, "float")) { + vecType = OPIHI_FLT; + } + + // generate the needed vectors + for (j = 0; j < Nval; j++) { + vec[Nvec + j] = InitVector(); + if (Nval == 1) { + strcpy (vec[Nvec + j]->name, name); + } else { + snprintf (vec[Nvec + j]->name, OPIHI_NAME_SIZE, "%s:%d", name, j); + } + ResetVector (vec[Nvec + j], vecType, MAX (Nrows,1)); + } + + // read the actual table data into a column + void *data; + status = gfits_get_bintable_column (&header, &ftable, name, &data); + assert (status); + +# define ASSIGN_DATA(TYPE,OPTYPE) \ + /* assign the data to the actual vector */ \ + if (!strcmp (type, #TYPE)) { \ + TYPE *Ptr = data; \ + for (k = 0; k < Nrows; k++) { \ + for (j = 0; j < Nval; j++, Ptr++) { \ + vec[Nvec + j][0].elements.OPTYPE[k] = *Ptr; \ + } } } + + // assign the data to the actual vector + ASSIGN_DATA(char, Int); + ASSIGN_DATA(short, Int); + ASSIGN_DATA(int, Int); + ASSIGN_DATA(int64_t, Int); + ASSIGN_DATA(float, Flt); + ASSIGN_DATA(double, Flt); + + free (data); + Nvec += Nval; + } + + gfits_free_header (&theader); + gfits_free_table (&ftable); + + fclose (f); + + *nvec = Nvec; + return vec; + +//escape: +// gfits_free_header (&header); +// gfits_free_matrix (&matrix); +// gfits_free_header (&theader); +// gfits_free_table (&ftable); +// +// fclose (f); +// fflush (f); +// return (FALSE); +} Index: /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorOps.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorOps.c (revision 33308) +++ /branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorOps.c (revision 33309) @@ -15,15 +15,22 @@ // this function is NOT thread protected : it is only used in startup and/or shutdown +void FreeVectorArray (Vector **vec, int Nvec) { + + int i; + + if (!vec) return; + for (i = 0; i < Nvec; i++) { + if (!vec[i]) continue; + if (vec[i]->elements.Int) { + free (vec[i]->elements.Int); + } + free (vec[i]); + } + free (vec); +} + +// this function is NOT thread protected : it is only used in startup and/or shutdown void FreeVectors () { - - int i; - - for (i = 0; i < Nvectors; i++) { - if (vectors[i][0].elements.Int) { - free (vectors[i][0].elements.Int); - } - free (vectors[i]); - } - free (vectors); + FreeVectorArray (vectors, Nvectors); } @@ -93,4 +100,34 @@ } +// Assign the given Vector to the internal array of vectors by name +int AssignVector (Vector *vec, char *name, int mode, int verbose) { + + int i; + + if (name == NULL) goto error; + if (ISNUM(name[0])) goto error; + if (IsBuffer(name)) goto error; + + for (i = 0; (i < Nvectors) && (strcmp(vectors[i][0].name, name)); i++); + /* is a new vector */ + if (i == Nvectors) { + if (mode == OLDVECTOR) goto error; + pthread_mutex_lock (&mutex); + Nvectors ++; + REALLOCATE (vectors, Vector *, Nvectors); + vectors[i] = vec; + pthread_mutex_unlock (&mutex); + return TRUE; + } + /* is an old vector */ + if (mode == NEWVECTOR) goto error; + vectors[i] = vec; + return TRUE; + + error: + if (verbose) gprint (GP_ERR, "invalid vector %s\n", name); + return FALSE; +} + /* delete by pointer */ int DeleteVector (Vector *vec) { @@ -287,118 +324,2 @@ return (TRUE); } - -// write a set of vectors to a FITS file -int WriteVectorTableFITS (char *filename, char *extname, Vector **vec, int Nvec, int append, char *format) { - - char *tformat = NULL; - Header header; - Matrix matrix; - Header theader; - FTable ftable; - - int j; - FILE *f = NULL; - - /* open file for outuput */ - if (append) { - f = fopen (filename, "a"); - } else { - f = fopen (filename, "w"); - } - if (f == (FILE *) NULL) { - gprint (GP_ERR, "can't open file for write\n"); - return (FALSE); - } - - if (!append) { - gfits_init_header (&header); - header.extend = TRUE; - gfits_create_header (&header); - gfits_create_matrix (&header, &matrix); - } - - gfits_create_table_header (&theader, "BINTABLE", extname); - - ALLOCATE (tformat, char, 2*Nvec); - if (format) { - // the bintable format string can only define the byte-width of each field. valid output columns are currently: - // B (char), I (16 bit short), J (32 bit int), E (32 bit float), D (64 bit double). - // the format string is just the sequence of types, eg: LIIJEED - // validate the format string - char *ptr = format; - for (j = 0; j < Nvec; j++) { - while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++; - if (*ptr == 0) { - gprint (GP_ERR, "error in binary table format %s (insufficient format chars)\n", format); - goto escape; - } - if ((*ptr != 'B') && (*ptr != 'I') && (*ptr != 'J') && (*ptr != 'D') && (*ptr != 'E')) { - gprint (GP_ERR, "error in binary table format %s: invalid character %c\n", format, *ptr); - goto escape; - } - tformat[2*j + 0] = *ptr; - tformat[2*j + 1] = 0; // a bit sleazy : use a 2xN string to store N 1-byte strings - ptr ++; - } - while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++; - if (*ptr) { - gprint (GP_ERR, "error in binary table format %s (extra characters in format)\n", format); - goto escape; - } - } else { - for (j = 0; j < Nvec; j++) { - // if the format is not defined, just use the native byte-widths - tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'J'; - tformat[2*j + 1] = 0; - } - } - - // define the columns of the table. XXX NOTE: we cannot have duplicate names in - // output table (because the data goes to the named column below). need to enforce - // this somehow - for (j = 0; j < Nvec; j++) { - gfits_define_bintable_column (&theader, &tformat[2*j], vec[j][0].name, NULL, NULL, 1.0, 0.0); - } - free (tformat); - - // generate the output array that carries the data - gfits_create_table (&theader, &ftable); - - // add the vectors to the output array - for (j = 0; j < Nvec; j++) { - if (vec[j][0].type == OPIHI_FLT) { - gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "double", vec[j][0].elements.Flt, vec[j][0].Nelements); - } else { - gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "int", vec[j][0].elements.Int, vec[j][0].Nelements); - } - } - - if (!append) { - gfits_fwrite_header (f, &header); - gfits_fwrite_matrix (f, &matrix); - gfits_free_header (&header); - gfits_free_matrix (&matrix); - } - gfits_fwrite_Theader (f, &theader); - gfits_fwrite_table (f, &ftable); - - gfits_free_header (&theader); - gfits_free_table (&ftable); - - fclose (f); - fflush (f); - return (TRUE); - - escape: - if (!append) { - gfits_free_header (&header); - gfits_free_matrix (&matrix); - } - gfits_free_header (&theader); - gfits_free_table (&ftable); - - if (tformat) free (tformat); - fclose (f); - fflush (f); - return (FALSE); -}