Index: trunk/Ohana/src/libdvo/include/dvo.h =================================================================== --- trunk/Ohana/src/libdvo/include/dvo.h (revision 38154) +++ trunk/Ohana/src/libdvo/include/dvo.h (revision 38441) @@ -10,9 +10,12 @@ /* DVO table modes */ -typedef enum {DVO_MODE_UNDEF, DVO_MODE_RAW, DVO_MODE_MEF, DVO_MODE_SPLIT, DVO_MODE_MYSQL} DVOTableMode; +typedef enum {DVO_MODE_UNDEF = 0, DVO_MODE_RAW, DVO_MODE_MEF, DVO_MODE_SPLIT, DVO_MODE_MYSQL} DVOCatMode; + +/* DVO table modes */ +typedef enum {DVO_COMPRESS_NONE = 0, DVO_COMPRESS_NONE_1, DVO_COMPRESS_NONE_2, DVO_COMPRESS_AUTO, DVO_COMPRESS_GZIP_1, DVO_COMPRESS_GZIP_2, DVO_COMPRESS_RICE_1} DVOCatCompress; // /* DVO table formats */ typedef enum { - DVO_FORMAT_UNDEF, + DVO_FORMAT_UNDEF = 0, DVO_FORMAT_INTERNAL, DVO_FORMAT_ELIXIR, @@ -30,11 +33,32 @@ DVO_FORMAT_PS1_V4, DVO_FORMAT_PS1_V5, -} DVOTableFormat; +} DVOCatFormat; + +typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes; + +/* catalog values to be loaded */ +typedef enum { + DVO_LOAD_NONE = 0x0000, + DVO_LOAD_AVERAGE = 0x0001, + DVO_LOAD_MEASURE = 0x0002, + DVO_LOAD_MISSING = 0x0004, + DVO_LOAD_SECFILT = 0x0008, + DVO_SKIP_AVERAGE = 0x0010, + DVO_SKIP_MEASURE = 0x0020, + DVO_SKIP_MISSING = 0x0040, + DVO_SKIP_SECFILT = 0x0080, + DVO_LOAD_LENSING = 0x0100, + DVO_LOAD_LENSOBJ = 0x0200, + DVO_SKIP_LENSING = 0x0400, + DVO_SKIP_LENSOBJ = 0x0800, + DVO_LOAD_STARPAR = 0x1000, + DVO_SKIP_STARPAR = 0x2000, + DVO_LOAD_GALPHOT = 0x4000, + DVO_SKIP_GALPHOT = 0x8000, +} DVOCatFlags; /* image data modes in RegImage */ typedef enum {T_UNDEF = -1, T_NONE, T_OBJECT, T_DARK, T_BIAS, T_FLAT, T_MASK, T_FRINGE, T_SCATTER, T_MODES, T_FRINGEPTS, T_ANY, N_TYPE} ElixirDetrendTypes; typedef enum {M_UNDEF = -1, M_NONE, M_MEF, M_SPLIT, M_SINGLE, M_CUBE, M_SLICE, M_MODES, N_MODE} ElixirDetrendModes; - -typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes; // these are used as NAN for types of int values @@ -54,28 +78,13 @@ # define IMREG_DIST 0x01 /* image distributed, only imregister-3.0 */ -/* catalog values to be loaded */ -# define LOAD_NONE 0x0000 -# define LOAD_AVES 0x0001 -# define LOAD_MEAS 0x0002 -# define LOAD_MISS 0x0004 -# define LOAD_SECF 0x0008 -# define SKIP_AVES 0x0010 -# define SKIP_MEAS 0x0020 -# define SKIP_MISS 0x0040 -# define SKIP_SECF 0x0080 -# define LOAD_LENSING 0x0100 -# define LOAD_LENSOBJ 0x0200 -# define SKIP_LENSING 0x0400 -# define SKIP_LENSOBJ 0x0800 -# define LOAD_STARPAR 0x1000 -# define SKIP_STARPAR 0x2000 - /* photometry code types */ // # define PHOT_PRI 0x01 -# define PHOT_SEC 0x02 -# define PHOT_DEP 0x03 -# define PHOT_REF 0x04 -# define PHOT_ALT 0x05 /* never stored, only for look-ups */ -# define PHOT_MAG 0x06 /* generic magnitude; never stored */ +typedef enum { + PHOT_SEC = 0x02, + PHOT_DEP = 0x03, + PHOT_REF = 0x04, + PHOT_ALT = 0x05, /* never stored, only for look-ups */ + PHOT_MAG = 0x06, /* generic magnitude; never stored */ +} DVOPhotCodeTypes; /* Image.code values -- these values are 32 bit (as of PS1_V1) */ @@ -222,7 +231,8 @@ char lockstate; char mode; /* what data storage mode is used for disk file? */ - char format; /* what data format is used for disk file? */ + DVOCatFormat format; /* what data format is used for disk file? */ char virtual; /* is table in ftable or vtable? */ - char swapped; /* is table in internal byte-order? */ + char nativeOrder; /* is table in internal byte-order? */ + char scaledValue; /* is table in internal byte-order? */ } FITS_DB; @@ -484,5 +494,5 @@ unsigned short Nmeasure; // number of psf measurements unsigned short Nmissing; // number of missings - unsigned short Nextend; // number of extended measurements + unsigned short Ngalphot; // number of extended measurements uint32_t measureOffset; // offset to first psf measurement uint32_t missingOffset; // offset to first missing obs @@ -704,10 +714,10 @@ unsigned short Nlensing; // number of lensing measurements unsigned short Nlensobj; // number of lensing measurements - unsigned short Nextend; // number of extended measurements + unsigned short Ngalphot; // number of galphot measurements int measureOffset; // offset to first psf measurement int missingOffset; // offset to first missing obs int lensingOffset; // offset to first lensing obs int lensobjOffset; // offset to mean lensing data - int extendOffset; // offset to extended object entry + int galphotOffset; // offset to extended object entry int starparOffset; // offset to stellar parameter data float refColorBlue; // color of astrometry ref stars @@ -742,12 +752,13 @@ // lensing data (optional?) - Lensing *lensing; - Lensobj *lensobj; - StarPar *starpar; + Lensing *lensing; + Lensobj *lensobj; + StarPar *starpar; + GalPhot *galphot; int Nsecfilt; /* number of secfilt entries for each average entry */ - off_t Naverage, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Nsecf_mem; /* current number of each component in memory */ - off_t Naves_disk, Nmeas_disk, Nmiss_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Nsecf_disk; /* current number of each component on disk */ - off_t Naves_off, Nmeas_off, Nmiss_off, Nlensing_off, Nlensobj_off, Nstarpar_off, Nsecf_off; /* index of first loaded data value */ + off_t Naverage, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Ngalphot, Nsecfilt_mem; /* current number of each component in memory */ + off_t Naverage_disk, Nmeasure_disk, Nmissing_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Ngalphot_disk, Nsecfilt_disk; /* current number of each component on disk */ + off_t Naverage_off, Nmeasure_off, Nmissing_off, Nlensing_off, Nlensobj_off, Nstarpar_off, Ngalphot_off, Nsecfilt_off; /* index of first loaded data value */ // note that we use these for the full-sky relphot analysis @@ -765,10 +776,11 @@ /* pointers to split data files */ - struct Catalog *measure_catalog; /* measure catalog data (split) */ - struct Catalog *missing_catalog; /* missing catalog data (split) */ - struct Catalog *secfilt_catalog; /* secfilt catalog data (split) */ - struct Catalog *lensing_catalog; /* lensing catalog data (split) */ - struct Catalog *lensobj_catalog; /* lensobj catalog data (split) */ - struct Catalog *starpar_catalog; /* starpar catalog data (split) */ + struct Catalog *measure_catalog; /* measure catalog data (split) */ + struct Catalog *missing_catalog; /* missing catalog data (split) */ + struct Catalog *secfilt_catalog; /* secfilt catalog data (split) */ + struct Catalog *lensing_catalog; /* lensing catalog data (split) */ + struct Catalog *lensobj_catalog; /* lensobj catalog data (split) */ + struct Catalog *starpar_catalog; /* starpar catalog data (split) */ + struct Catalog *galphot_catalog; /* galphot catalog data (split) */ unsigned int objID; @@ -777,9 +789,11 @@ /* extra catalog information */ char lockmode; - char catmode; /* storage mode (raw, mef, split, mysql) */ - char catformat; /* storage format (elixir, panstarrs, etc) */ + DVOCatMode catmode; /* storage mode (raw, mef, split, mysql) */ + DVOCatFormat catformat; /* storage format (elixir, panstarrs, etc) */ + DVOCatCompress catcompress; // output compress mode + DVOCatFlags catflags; /* choices to be loaded */ + int sorted; /* is measure table average-sorted? (NOTE this is an int only because gfits_scan %t requires it) */ - - short catflags; /* choices to be loaded */ + /* pointers for data manipulation */ @@ -928,6 +942,8 @@ int dvo_catalog_save_complete (Catalog *catalog, char VERBOSE); int dvo_catalog_update (Catalog *catalog, char VERBOSE); -int dvo_catalog_catformat (char *catformat); -int dvo_catalog_catmode (char *catmode); +DVOCatFormat dvo_catalog_catformat (char *catformat); +DVOCatMode dvo_catalog_catmode (char *catmode); +DVOCatCompress dvo_catalog_catcompress (char *catcompress); +char *dvo_catalog_compress_string (DVOCatCompress catcompress); void dvo_catalog_test (Catalog *catalog, int halt); @@ -956,22 +972,28 @@ int WriteRawSecFilt (FILE *f, SecFilt *secfilt, off_t Nsecfilt, char format); -DVOTableFormat FtableGetFormat (FTable *ftable); - -Average *FtableToAverage (FTable *ftable, off_t *Naverage, char *format, SecFilt **primary); -Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, char *format); -SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, char *format); -Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, char *format); -Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, char *format); -StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, char *format); -int FtableToImage (FTable *ftable, Header *theader, char *format); - -int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, char format, SecFilt *primary); -int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, char format); -int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, char format); -int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, char format); -int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, char format); -int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, char format); -int ImageToFtable (FTable *ftable, Header *theader, char format); -int ImageToVtable (VTable *vtable, Header *theader, char format); +DVOCatFormat FtableGetFormat (FTable *ftable); + +Average *FtableToAverage (FTable *ftable, off_t *Naverage, DVOCatFormat *format, SecFilt **primary, char nativeOrder); + +Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeOrder); +Missing *FtableToMissing (FTable *ftable, Average *average, off_t *Nmissing, DVOCatFormat *format, char nativeBytes); +SecFilt *FtableToSecFilt (FTable *ftable, Average *average, off_t *Nsecfilt, DVOCatFormat *format, char nativeOrder); +Lensing *FtableToLensing (FTable *ftable, Average *average, off_t *Nlensing, DVOCatFormat *format, char nativeOrder); +Lensobj *FtableToLensobj (FTable *ftable, Average *average, off_t *Nlensobj, DVOCatFormat *format, char nativeOrder); +StarPar *FtableToStarPar (FTable *ftable, Average *average, off_t *Nstarpar, DVOCatFormat *format, char nativeOrder); +GalPhot *FtableToGalPhot (FTable *ftable, Average *average, off_t *Ngalphot, DVOCatFormat *format, char nativeOrder); + +int FtableToImage (FTable *ftable, Header *theader, DVOCatFormat *format); + +int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative); +int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, DVOCatFormat format, SecFilt *primary, int swapFromNative); +int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, DVOCatFormat format, int swapFromNative); +int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, DVOCatFormat format, int swapFromNative); +int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, DVOCatFormat format, int swapFromNative); +int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, DVOCatFormat format, int swapFromNative); +int GalPhotToFtable (FTable *ftable, GalPhot *galphot, off_t Ngalphot, DVOCatFormat format, int swapFromNative); + +int ImageToFtable (FTable *ftable, Header *theader, DVOCatFormat format); +int ImageToVtable (VTable *vtable, Header *theader, DVOCatFormat format); # include "loneos_defs.h" @@ -1005,5 +1027,5 @@ int gfits_table_set_Image (FTable *ftable); int gfits_table_mkheader_Image (Header *header); -Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *swapped); +Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *scaledValue, char *nativeOrder); /* flatcorr APIs */ @@ -1069,5 +1091,7 @@ void InitHost (HostInfo *host); -HostTableGroup *HostTableGroups (HostTable *table, int *ngroups); +HostTableGroup *HostTableGroupsUniqueMachines (HostTable *table, int *ngroups); +HostTableGroup *HostTableGroupsMaxNumber (HostTable *table, int *ngroups, int Nmax); + int HostTableGroupWaitJobsGetIO (HostTableGroup *table, char *file, int lineno, int VERBOSE); @@ -1106,4 +1130,5 @@ void dvo_lensobj_init (Lensobj *lensobj, int toZero); void dvo_starpar_init (StarPar *starpar); +void dvo_galphot_init (GalPhot *galphot); void InitRegionHosts (RegionHostInfo *hosts, int Nhosts, int NHOSTS);