Index: trunk/Ohana/src/opihi/cmd.data/cast.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/cast.c (revision 38255) +++ trunk/Ohana/src/opihi/cmd.data/cast.c (revision 38441) @@ -17,7 +17,15 @@ // out type char outType = OPIHI_NOTYPE; - if (!strcasecmp(argv[5], "int")) outType = OPIHI_INT; - if (!strcasecmp(argv[5], "float")) outType = OPIHI_FLT; + if (!strcasecmp(argv[5], "int")) outType = OPIHI_INT; + if (!strcasecmp(argv[5], "float")) outType = OPIHI_FLT; + if (!strcasecmp(argv[5], "single")) outType = OPIHI_FLT; + if (!strcasecmp(argv[5], "int64")) outType = OPIHI_FLT; if (outType == OPIHI_NOTYPE) goto usage; + + int ForceSingle = FALSE; + if (!strcasecmp(argv[5], "single")) ForceSingle = TRUE; + + int ForceI64 = FALSE; + if (!strcasecmp(argv[5], "int64")) ForceI64 = TRUE; if ((ovec = SelectVector (argv[1], ANYVECTOR, TRUE)) == NULL) goto error; @@ -32,5 +40,13 @@ opihi_flt *vo = ovec[0].elements.Flt; for (i = 0; i < ovec[0].Nelements; i++, vi++, vo++) { - *vo = *vi; + if (ForceSingle) { + float tmp = *vi; + *vo = tmp; + } else if (ForceI64) { + int64_t tmp = *vi; + *vo = tmp; + } else { + *vo = *vi; + } } return (TRUE); @@ -48,5 +64,10 @@ opihi_flt *vo = ovec[0].elements.Flt; for (i = 0; i < ovec[0].Nelements; i++, vi++, vo++) { - *vo = *vi; + if (ForceSingle) { + float tmp = *vi; + *vo = tmp; + } else { + *vo = *vi; + } } return (TRUE); @@ -66,5 +87,6 @@ usage: - gprint (GP_ERR, "SYNTAX: cast vec = vec as (int/float)\n"); + gprint (GP_ERR, "SYNTAX: cast vec = vec as (int/float/single)\n"); + gprint (GP_ERR, " note: (single) forces vector to have single precision\n"); return (FALSE); } Index: trunk/Ohana/src/opihi/cmd.data/rd.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/rd.c (revision 38255) +++ trunk/Ohana/src/opihi/cmd.data/rd.c (revision 38441) @@ -9,4 +9,10 @@ Buffer *buf; + int VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; + } + int JustHead = FALSE; if ((N = get_argument (argc, argv, "-head"))) { @@ -91,5 +97,5 @@ return (FALSE); } - if (gfits_extension_is_compressed (&buf[0].header)) { + if (gfits_extension_is_compressed_image (&buf[0].header)) { IsCompressed = TRUE; } @@ -124,5 +130,5 @@ Nword = 1; IsCompressed = FALSE; - if (gfits_extension_is_compressed (&buf[0].header)) { + if (gfits_extension_is_compressed_image (&buf[0].header)) { if (!strcmp (CCDKeyword, "EXTNAME")) Nword = 1; IsCompressed = TRUE; @@ -198,9 +204,14 @@ ftable.header[0].buffer = NULL; gfits_copy_header (&buf[0].header, ftable.header); - status = gfits_fread_ftable_data (f, &ftable, FALSE); // this just reads the bytes (not even a SWAP) - status = gfits_uncompress_image (&buf[0].header, &buf[0].matrix, &ftable); + + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { gprint (GP_ERR, "problem reading compressed image table data\n"); fclose (f); return FALSE; } + if (!gfits_byteswap_varlength_column (&ftable, 1)) { gprint (GP_ERR, "problem doing byteswap on compressed image metadata column\n"); fclose (f); gfits_free_table (&ftable); return FALSE; } + if (!gfits_uncompress_image (&buf[0].header, &buf[0].matrix, &ftable)) { gprint (GP_ERR, "problem uncompressing the image data\n"); fclose (f); gfits_free_table (&ftable); return FALSE; } + // uncompressing the image leaves the format as an extension gfits_extended_to_primary (&buf[0].header, TRUE, "Standard FITS"); gfits_free_table (&ftable); + + status = TRUE; // XXX this currently does not work for a cube (we get a cube back, not a specific plane) } else { @@ -239,6 +250,5 @@ buf[0].unsign = buf[0].header.unsign; - gprint (GP_LOG, "read "OFF_T_FMT" bytes from %s into buffer %s\n", - buf[0].header.datasize + buf[0].matrix.datasize, argv[2], argv[1]); + if (VERBOSE) gprint (GP_LOG, "read "OFF_T_FMT" bytes from %s into buffer %s\n", buf[0].header.datasize + buf[0].matrix.datasize, argv[2], argv[1]); blank = 0xffff; Index: trunk/Ohana/src/opihi/cmd.data/read_vectors.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/read_vectors.c (revision 38255) +++ trunk/Ohana/src/opihi/cmd.data/read_vectors.c (revision 38441) @@ -1,3 +1,5 @@ # include "data.h" +int dump_raw_table (FTable *table, char *message); +int dump_cmp_table (FTable *table, char *message); FILE *f = (FILE *) NULL; @@ -412,5 +414,5 @@ table.header = &header; - /* load appropriate extension (if extname is a number, use count) */ + /**** find the appropriate extension and read header (if extname is a number, use count) ****/ if (Nextend > -1) { // first extension is PHU, cannot be a table. @@ -429,28 +431,4 @@ } if (!gfits_load_header (f, &header)) ESCAPE ("error reading header for extension %d\n", Nextend); - - if (getSizes) { - read_table_sizes (&header); - if (CCDKeyword != NULL) free (CCDKeyword); - gfits_free_header (&header); - return TRUE; - } - - if (Nrows == -1) { - Nrows = header.Naxis[1] - start; - } - if (start < 0) ESCAPE ("invalid range: start < 0\n"); - if (start >= header.Naxis[1]) ESCAPE ("invalid range: start >= Ny (%d)\n", header.Naxis[1]); - if (Nrows < 0) ESCAPE ("invalid range: Nrows < 0\n"); - - // just a warning: - if (start + Nrows > header.Naxis[1]) { - if (VERBOSE) gprint (GP_ERR, "NOTE: reading last block will return only %d rows\n", header.Naxis[1] - start); - } - - // Ny = 100, start = 0, Nrows = -1 -> Nrows => 100 - // Ny = 100, start = 10, Nrows = 90 - - if (!gfits_fread_ftable_range (f, padIfShort, &table, start, Nrows)) ESCAPE ("error reading table for extension %d\n", Nextend); } else { if (CCDKeyword == NULL) { @@ -460,5 +438,4 @@ CCDKeyword = strcreate ("EXTNAME"); } - while (1) { if (!gfits_load_header (f, &header)) { @@ -469,5 +446,4 @@ return (TRUE); } - Nbytes = gfits_data_size (&header); @@ -482,30 +458,48 @@ continue; } - - if (getSizes) { - read_table_sizes (&header); - if (CCDKeyword != NULL) free (CCDKeyword); - gfits_free_header (&header); - return TRUE; - } - - if (Nrows == -1) { - Nrows = header.Naxis[1] - start; - } - if (start < 0) ESCAPE ("invalid range: start < 0\n"); - if (start >= header.Naxis[1]) ESCAPE ("invalid range: start >= Ny (%d)\n", header.Naxis[1]); - if (Nrows < 0) ESCAPE ("invalid range: Nrows < 0\n"); - - // just a warning: - if (start + Nrows > header.Naxis[1]) { - if (VERBOSE) gprint (GP_ERR, "NOTE: reading last block will return only %d rows\n", header.Naxis[1] - start); - } - - if (!gfits_fread_ftable_range (f, padIfShort, &table, start, Nrows)) ESCAPE ("error reading table for extension %d\n", Nextend); - - // if (!gfits_fread_ftable_data (f, &table, padIfShort)) ESCAPE ("error reading table for extension\n"); - break; } + } + + int IsCompressed = gfits_extension_is_compressed_table (&header); + + if (getSizes && !IsCompressed) { + read_table_sizes (&header); + if (CCDKeyword != NULL) free (CCDKeyword); + gfits_free_header (&header); + return TRUE; + } + if (IsCompressed) { + if (getSizes) { + gprint (GP_ERR, "%s[%s] is compressed: -sizes is invalid\n", filename, extname); + gfits_free_header (&header); + return FALSE; + } + if ((start > 0) || (Nrows > -1)) { + gprint (GP_ERR, "%s[%s] is compressed: must read entire table\n", filename, extname); + gfits_free_header (&header); + return FALSE; + } + } + + if (Nrows == -1) { + Nrows = header.Naxis[1] - start; + } + if (start < 0) ESCAPE ("invalid range: start < 0\n"); + if (start >= header.Naxis[1]) ESCAPE ("invalid range: start >= Ny ("OFF_T_FMT")\n", header.Naxis[1]); + if (Nrows < 0) ESCAPE ("invalid range: Nrows < 0\n"); + + // just a warning: + if (start + Nrows > header.Naxis[1]) { + if (VERBOSE) gprint (GP_ERR, "NOTE: reading last block will return only "OFF_T_FMT" rows\n", header.Naxis[1] - start); + } + + // Ny = 100, start = 0, Nrows = -1 -> Nrows => 100 + // Ny = 100, start = 10, Nrows = 90 + + if (IsCompressed) { + if (!gfits_fread_ftable_data (f, &table, padIfShort)) ESCAPE ("error reading table for extension %d\n", Nextend); + } else { + if (!gfits_fread_ftable_range (f, padIfShort, &table, start, Nrows)) ESCAPE ("error reading table for extension %d\n", Nextend); } @@ -518,4 +512,29 @@ Binary = !strcmp (type, "BINTABLE"); Ny = header.Naxis[1]; + + Header *outheader = &header; + FTable *outtable = &table; + + Header rawheader; + FTable rawtable; + if (IsCompressed) { + rawtable.header = &rawheader; + + dump_cmp_table (&table, "rd cmp"); + + int Nfields; + if (!gfits_scan (&header, "TFIELDS", "%d", 1, &Nfields)) ESCAPE ("cannot find TFIELDS in header\n"); + for (i = 0; i < Nfields; i++) { + gfits_byteswap_varlength_column (&table, i+1); + } + if (!gfits_uncompress_table (&table, &rawtable)) ESCAPE ("failed to uncompress table\n"); + dump_raw_table (&rawtable, "rd raw"); + + outheader = &rawheader; + outtable = &rawtable; + gfits_free_header (&header); + gfits_free_table (&table); + Ny = rawheader.Naxis[1]; + } /* find columns which match requested vectors */ @@ -527,14 +546,14 @@ Nval = 0; if (Binary) { - if (!gfits_get_bintable_column_type (&header, argv[i], type, &Nval)) ESCAPE ("requested field not found"); - if (!gfits_get_bintable_column (&header, &table, argv[i], &data)) ESCAPE ("error reading data from specified field"); + if (!gfits_get_bintable_column_type (outheader, argv[i], type, &Nval)) ESCAPE ("requested field not found\n"); + if (!gfits_get_bintable_column_raw (outheader, outtable, argv[i], &data, IsCompressed)) ESCAPE ("error reading data from specified field\n"); } else { - if (!gfits_get_table_column_type (&header, argv[i], type)) ESCAPE ("requested field not found"); - if (!gfits_get_table_column (&header, &table, argv[i], &data)) ESCAPE ("error reading data from specified field"); - } - if (Nval == 0) ESCAPE ("no data for field in table"); + if (!gfits_get_table_column_type (outheader, argv[i], type)) ESCAPE ("requested field not found\n"); + if (!gfits_get_table_column (outheader, outtable, argv[i], &data)) ESCAPE ("error reading data from specified field\n"); + } + if (Nval == 0) ESCAPE ("no data for field in table\n"); vecType = OPIHI_INT; - if (!strcmp (type, "double") || !strcmp (type, "float")) { + if (!strcmp (type, "double") || !strcmp (type, "float") || !strcmp (type, "int64_t")) { vecType = OPIHI_FLT; } @@ -570,9 +589,9 @@ else sprintf (name, "%s:%d", argv[i], j); - if ((vec[j] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) ESCAPE ("bad vector name"); + if ((vec[j] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) ESCAPE ("bad vector name\n"); ResetVector (vec[j], vecType, Ny); } - if (!VectorAssignData (vec, type, data, Ny, Nval)) ESCAPE ("bad column type %s", type); + if (!VectorAssignData (vec, type, data, Ny, Nval)) ESCAPE ("bad column type %s\n", type); } else { @@ -584,9 +603,9 @@ else sprintf (name, "%s:%d", argv[i], j); - if ((vec[j] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) ESCAPE ("bad vector name"); + if ((vec[j] = SelectVector (name, ANYVECTOR, TRUE)) == NULL) ESCAPE ("bad vector name\n"); ResetVector (vec[j], vecType, Nval); } - if (!VectorAssignDataTranspose (vec, type, data, Ny, Nval)) ESCAPE ("bad column type %s", type); + if (!VectorAssignDataTranspose (vec, type, data, Ny, Nval)) ESCAPE ("bad column type %s\n", type); } free (data); @@ -594,6 +613,6 @@ } if (CCDKeyword != NULL) free (CCDKeyword); - gfits_free_table (&table); - gfits_free_header (&header); + gfits_free_table (outtable); + gfits_free_header (outheader); return (TRUE); } Index: trunk/Ohana/src/opihi/cmd.data/tv.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/tv.c (revision 38255) +++ trunk/Ohana/src/opihi/cmd.data/tv.c (revision 38441) @@ -71,5 +71,5 @@ int tooBig = buf[0].matrix.Naxis[2] ? (plane >= buf[0].matrix.Naxis[2]) : plane > 0; if (tooBig) { - gprint (GP_ERR, " ERROR: -plane (plane) : out of bounds (%d vs %d)\n", plane, buf[0].matrix.Naxis[2]); + gprint (GP_ERR, " ERROR: -plane (plane) : out of bounds (%d vs "OFF_T_FMT")\n", plane, buf[0].matrix.Naxis[2]); return (FALSE); } Index: trunk/Ohana/src/opihi/cmd.data/vstats.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/vstats.c (revision 38255) +++ trunk/Ohana/src/opihi/cmd.data/vstats.c (revision 38441) @@ -121,4 +121,6 @@ goto skip; } + // we can hit inf if pmax and pmin are close to the max range + if (isinf(dx)) dx = DBL_MAX / Nbin; ALLOCATE (Nval, int, Nbin + 2); Index: trunk/Ohana/src/opihi/cmd.data/wd.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/wd.c (revision 38255) +++ trunk/Ohana/src/opihi/cmd.data/wd.c (revision 38441) @@ -18,4 +18,14 @@ } + char *CompressMode = NULL; + if ((N = get_argument (argc, argv, "-compress-mode"))) { + remove_argument (N, &argc, argv); + CompressMode = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } else if ((N = get_argument (argc, argv, "-compress"))) { + remove_argument (N, &argc, argv); + CompressMode = strcreate("GZIP_1"); + } + outZero = 0; newZero = FALSE; @@ -54,4 +64,5 @@ if (outUnsign == -1) { gprint (GP_ERR, "-unsign options: t, f, true, false\n"); + FREE (CompressMode); return (FALSE); } @@ -62,8 +73,9 @@ if (argc != 3) { gprint (GP_ERR, "USAGE: wd [-bitpix N] [-bscale X] [-bzero X] [-extend] [-newplane]\n"); + FREE (CompressMode); return (FALSE); } - if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) { FREE (CompressMode); return (FALSE); } if (!newBitpix) outBitpix = buf[0].bitpix; @@ -159,7 +171,46 @@ gfits_free_header (&temp_header); gfits_free_matrix (&temp_matrix); + FREE (CompressMode); return (status); } + // not compatible with extend + if (CompressMode) { + Header myHeader; + Matrix myMatrix; + + FILE *f = fopen (argv[2], "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file for output %s\n", argv[2]); + FREE (CompressMode); + return FALSE; + } + + gfits_init_header (&myHeader); + myHeader.extend = TRUE; + gfits_create_header (&myHeader); + gfits_create_matrix (&myHeader, &myMatrix); + gfits_fwrite_header (f, &myHeader); + gfits_fwrite_matrix (f, &myMatrix); + gfits_free_header (&myHeader); + gfits_free_matrix (&myMatrix); + + FTable ftable; + Header theader; + + ftable.header = &theader; + + gfits_compress_image (&temp_header, &temp_matrix, &ftable, NULL, CompressMode); + gfits_byteswap_varlength_column (&ftable, 1); + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + fclose (f); + + gfits_free_header (&theader); + gfits_free_table (&ftable); + FREE (CompressMode); + return (TRUE); + } + /* the actual write-to-disk goes here */ if (!gfits_write_header (argv[2], &temp_header)) { @@ -167,4 +218,5 @@ gfits_free_header (&temp_header); gfits_free_matrix (&temp_matrix); + FREE (CompressMode); return (FALSE); } @@ -174,4 +226,5 @@ gfits_free_header (&temp_header); gfits_free_matrix (&temp_matrix); + FREE (CompressMode); return (FALSE); } @@ -180,4 +233,5 @@ gfits_free_matrix (&temp_matrix); + FREE (CompressMode); return (TRUE); } Index: trunk/Ohana/src/opihi/cmd.data/write_vectors.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/write_vectors.c (revision 38255) +++ trunk/Ohana/src/opihi/cmd.data/write_vectors.c (revision 38441) @@ -3,9 +3,8 @@ int write_vectors (int argc, char **argv) { - int append; int i, j, Nvec, Ne, N; FILE *f; char **fmtlist, *fmttype; - char *p0, *p1, *p2, *format, *FITS; + char *p0, *p1, *p2, *format; Vector **vec; @@ -34,5 +33,5 @@ /* option generate a FITS output table */ - FITS = NULL; + char *FITS = NULL; if ((N = get_argument (argc, argv, "-fits"))) { remove_argument (N, &argc, argv); @@ -58,8 +57,21 @@ } - append = FALSE; + int append = FALSE; if ((N = get_argument (argc, argv, "-append"))) { remove_argument (N, &argc, argv); append = TRUE; + } + + char *compress = NULL; + if ((N = get_argument (argc, argv, "-compress-mode"))) { + remove_argument (N, &argc, argv); + compress = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } else if ((N = get_argument (argc, argv, "-compress"))) { + remove_argument (N, &argc, argv); + compress = strcreate("GZIP_1"); + if (!FITS) { + fprintf (stderr, "NOTE: write_vectors -compress has no effect on non-FITS\n"); + } } @@ -105,5 +117,5 @@ if (FITS) { - int status = WriteVectorTableFITS (argv[1], FITS, vec, Nvec, append, format); + int status = WriteVectorTableFITS (argv[1], FITS, vec, Nvec, append, compress, format); free (vec); return status; @@ -243,8 +255,8 @@ gprint (GP_ERR, " -csv : write a comma-separated values file (eg, to read in excel)\n"); gprint (GP_ERR, " -f \"format\" : provide formatting codes for output:\n"); - gprint (GP_ERR, " ascii / csv : format consists of c-style format codes in form %NN.Md\n"); + gprint (GP_ERR, " ascii / csv : format consists of c-style format codes in form %%NN.Md\n"); gprint (GP_ERR, ", the following codes are allowed\n"); - gprint (GP_ERR, " %e -- double, %f -- float\n"); - gprint (GP_ERR, " %d -- int, %c -- char, %x -- hex int\n"); + gprint (GP_ERR, " %%e -- double, %%f -- float\n"); + gprint (GP_ERR, " %%d -- int, %%c -- char, %%x -- hex int\n"); gprint (GP_ERR, " FITS : format consists of the following FITS binary table format codes:\n"); gprint (GP_ERR, " B : 1 byte int\n");