Index: trunk/Ohana/src/libdvo/Makefile =================================================================== --- trunk/Ohana/src/libdvo/Makefile (revision 33648) +++ trunk/Ohana/src/libdvo/Makefile (revision 33649) @@ -34,4 +34,5 @@ $(DESTINC)/ps1_v1_defs.h \ $(DESTINC)/ps1_v2_defs.h \ +$(DESTINC)/ps1_v3_defs.h \ $(DESTINC)/ps1_ref_defs.h @@ -43,4 +44,5 @@ $(SRC)/dvosorts.$(ARCH).o \ $(SRC)/dvo_photcode_ops.$(ARCH).o \ +$(SRC)/dvo_photcode_utils.$(ARCH).o \ $(SRC)/LoadPhotcodes.$(ARCH).o \ $(SRC)/LoadPhotcodesText.$(ARCH).o \ @@ -51,4 +53,5 @@ $(SRC)/mosaic_astrom.$(ARCH).o \ $(SRC)/fits_db.$(ARCH).o \ +$(SRC)/dvo_tiny_values.$(ARCH).o \ $(SRC)/dvo_image.$(ARCH).o \ $(SRC)/dvo_image_raw.$(ARCH).o \ @@ -69,5 +72,7 @@ $(SRC)/dvo_convert_PS1_V1.$(ARCH).o \ $(SRC)/dvo_convert_PS1_V2.$(ARCH).o \ +$(SRC)/dvo_convert_PS1_V3.$(ARCH).o \ $(SRC)/dvo_convert_PS1_REF.$(ARCH).o \ +$(SRC)/flatcorr_io.$(ARCH).o \ $(SRC)/skyregion_io.$(ARCH).o \ $(SRC)/skyregion_gsc.$(ARCH).o \ @@ -94,5 +99,6 @@ $(SRC)/match_image.$(ARCH).o \ $(SRC)/db_utils.$(ARCH).o \ -$(SRC)/convert.$(ARCH).o +$(SRC)/convert.$(ARCH).o \ +$(SRC)/HostTable.$(ARCH).o Index: trunk/Ohana/src/libdvo/include/dvo.h =================================================================== --- trunk/Ohana/src/libdvo/include/dvo.h (revision 33648) +++ trunk/Ohana/src/libdvo/include/dvo.h (revision 33649) @@ -26,4 +26,5 @@ DVO_FORMAT_PS1_V1, DVO_FORMAT_PS1_V2, + DVO_FORMAT_PS1_V3, } DVOTableFormat; @@ -95,34 +96,35 @@ /* Image.code values -- these values are 32 bit (as of PS1_V1) */ typedef enum { - ID_IMAGE_NEW = 0x00000000, /* no calibrations yet attempted */ - ID_IMAGE_PHOTOM_NOCAL = 0x00000001, /* user-set value used within relphot: ignore */ - ID_IMAGE_PHOTOM_POOR = 0x00000002, /* relphot says image is bad (dMcal > limit) */ - ID_IMAGE_PHOTOM_SKIP = 0x00000004, /* user-set value: assert that this image has bad photometry */ - ID_IMAGE_PHOTOM_FEW = 0x00000008, /* currently too few measurements for photometry */ - ID_IMAGE_ASTROM_NOCAL = 0x00000010, /* user-set value used within relastro: ignore */ - ID_IMAGE_ASTROM_POOR = 0x00000020, /* relastro says image is bad (dR,dD > limit) */ - ID_IMAGE_ASTROM_FAIL = 0x00000040, /* relastro fit diverged, fit not applied */ - ID_IMAGE_ASTROM_SKIP = 0x00000080, /* user-set value: assert that this image has bad astrometry */ - ID_IMAGE_ASTROM_FEW = 0x00000100, /* currently too few measurements for astrometry */ + ID_IMAGE_NEW = 0x00000000, /* no calibrations yet attempted */ + ID_IMAGE_PHOTOM_NOCAL = 0x00000001, /* user-set value used within relphot: ignore */ + ID_IMAGE_PHOTOM_POOR = 0x00000002, /* relphot says image is bad (dMcal > limit) */ + ID_IMAGE_PHOTOM_SKIP = 0x00000004, /* user-set value: assert that this image has bad photometry */ + ID_IMAGE_PHOTOM_FEW = 0x00000008, /* currently too few measurements for photometry */ + ID_IMAGE_ASTROM_NOCAL = 0x00000010, /* user-set value used within relastro: ignore */ + ID_IMAGE_ASTROM_POOR = 0x00000020, /* relastro says image is bad (dR,dD > limit) */ + ID_IMAGE_ASTROM_FAIL = 0x00000040, /* relastro fit diverged, fit not applied */ + ID_IMAGE_ASTROM_SKIP = 0x00000080, /* user-set value: assert that this image has bad astrometry */ + ID_IMAGE_ASTROM_FEW = 0x00000100, /* currently too few measurements for astrometry */ + ID_IMAGE_PHOTOM_UBERCAL = 0x00000200, /* externally-supplied photometry zero point from ubercal analysis */ } DVOImageFlags; /* Measure.flags values -- these values are 32 bit (as of PS1_V1) */ typedef enum { - ID_MEAS_NOCAL = 0x00000001, // detection ignored for this analysis (photcode, time range) -- internal only - ID_MEAS_POOR_PHOTOM = 0x00000002, // detection is photometry outlier - ID_MEAS_SKIP_PHOTOM = 0x00000004, // detection was ignored for photometry measurement - ID_MEAS_AREA = 0x00000008, // detection near image edge - ID_MEAS_POOR_ASTROM = 0x00000010, // detection is astrometry outlier - ID_MEAS_SKIP_ASTROM = 0x00000020, // detection was ignored for astrometry measurement - ID_MEAS_USED_OBJ = 0x00000040, // detection was used during opdate objects - ID_MEAS_USED_CHIP = 0x00000080, // detection was used during update chips - ID_MEAS_BLEND_MEAS = 0x00000100, // detection is within radius of multiple objects - ID_MEAS_BLEND_OBJ = 0x00000200, // multiple detections within radius of object - ID_MEAS_UNDEF_3 = 0x00000400, // unused - ID_MEAS_UNDEF_4 = 0x00000800, // unused - ID_MEAS_BLEND_MEAS_X = 0x00001000, // detection is within radius of multiple objects across catalogs - ID_MEAS_ARTIFACT = 0x00002000, // detection is thought to be non-astronomical - ID_MEAS_UNDEF_5 = 0x00004000, // unused - ID_MEAS_UNDEF_6 = 0x00008000, // unused + ID_MEAS_NOCAL = 0x00000001, // detection ignored for this analysis (photcode, time range) -- internal only + ID_MEAS_POOR_PHOTOM = 0x00000002, // detection is photometry outlier + ID_MEAS_SKIP_PHOTOM = 0x00000004, // detection was ignored for photometry measurement + ID_MEAS_AREA = 0x00000008, // detection near image edge + ID_MEAS_POOR_ASTROM = 0x00000010, // detection is astrometry outlier + ID_MEAS_SKIP_ASTROM = 0x00000020, // detection was ignored for astrometry measurement + ID_MEAS_USED_OBJ = 0x00000040, // detection was used during update objects + ID_MEAS_USED_CHIP = 0x00000080, // detection was used during update chips (XXX this probably does not make it into the db) + ID_MEAS_BLEND_MEAS = 0x00000100, // detection is within radius of multiple objects + ID_MEAS_BLEND_OBJ = 0x00000200, // multiple detections within radius of object + ID_MEAS_UNDEF_3 = 0x00000400, // unused + ID_MEAS_UNDEF_4 = 0x00000800, // unused + ID_MEAS_BLEND_MEAS_X = 0x00001000, // detection is within radius of multiple objects across catalogs + ID_MEAS_ARTIFACT = 0x00002000, // detection is thought to be non-astronomical + ID_MEAS_UNDEF_5 = 0x00004000, // unused + ID_MEAS_PHOTOM_UBERCAL = 0x00008000, // externally-supplied zero point from ubercal analysis } DVOMeasureFlags; @@ -141,5 +143,5 @@ */ -/* Average.code values -- these values are 32 bit (as of PS1_V1) */ +/* Average.flags values -- these values are 32 bit (as of PS1_V1) */ typedef enum { ID_STAR_FEW = 0x00000001, // used within relphot: skip star @@ -168,4 +170,18 @@ } DVOAverageFlags; +/* Secfilt.flags values -- these values are 32 bit (as of PS1_V1) */ +typedef enum { + ID_SECF_STAR_FEW = 0x00000001, // used within relphot: skip star + ID_SECF_STAR_POOR = 0x00000002, // used within relphot: skip star + ID_SECF_USE_SYNTH = 0x00000004, // synthetic photometry used in average measurement + ID_SECF_USE_UBERCAL = 0x00000008, // synthetic photometry used in average measurement + ID_PHOTOM_PASS_0 = 0x00000100, // average magnitude calculated in 0th pass + ID_PHOTOM_PASS_1 = 0x00000200, // average magnitude calculated in 1th pass + ID_PHOTOM_PASS_2 = 0x00000400, // average magnitude calculated in 2th pass + ID_PHOTOM_PASS_3 = 0x00000800, // average magnitude calculated in 3th pass + ID_PHOTOM_PASS_4 = 0x00001000, // average magnitude calculated in 4th pass + ID_SECF_OBJ_EXT = 0x01000000, // extended in this band +} DVOSecfiltFlags; + /*** general dvo structures (internal use only / not IO) ***/ @@ -196,4 +212,5 @@ char **filename; SkyRegion *regions; + char hosts[80]; } SkyTable; @@ -203,5 +220,54 @@ char **filename; SkyRegion **regions; + char hosts[80]; } SkyList; + +typedef struct { + off_t Nimage; + FlatCorrectionImage *image; + off_t Ncorr; + int *IDtoSeq; + FlatCorrection *corr; + float ***offset; // the correction images represented as a set of arrays (same sequence as *image) + int Nseason; + e_time *tstart; + e_time *tstop; +} FlatCorrectionTable; + +/* definitions for parallel dvo host information + XXX : need better names (safer namespace) + */ + +typedef enum { + DATA_ON_TGT = 0x01, + DATA_ON_BCK = 0x02, + DATA_USE_BCK = 0x04, + + DATA_COPY_FAILURE = 0x80, +} SkyTableDataFlags; + +typedef enum { + HOST_STDIN = 0, + HOST_STDOUT = 1, + HOST_STDERR = 2, +} HostInfoIOfd; + +typedef struct { + char *hostname; // name of remote machine + char *pathname; // name of directory for this machine's data + char *results; // name of file machine's result data + int hostID; // remove machine ID in SkyTable + int stdio[3]; // fd's for communication with the remote host + int pid; // remote process ID + int status; // job exit status + IOBuffer stdout; + IOBuffer stderr; +} HostInfo; + +typedef struct { + int Nhosts; + HostInfo *hosts; + unsigned short *index; +} HostTable; // special-case function: @@ -218,4 +284,19 @@ } PhotCodeData; +typedef enum { + DVO_TV_MEASURE = 0x01, + DVO_TV_AVERAGE = 0x02, +} DVOTinyValueMode; + +// XXX DROP? // a reduced-subset structure for relastro +// XXX DROP? typedef struct { +// XXX DROP? double R; +// XXX DROP? double D; +// XXX DROP? unsigned short Nmeasure; +// XXX DROP? int measureOffset; +// XXX DROP? uint32_t flags; +// XXX DROP? int catID; +// XXX DROP? } AverageTinyAstro; + // a reduced-subset structure for relphot typedef struct { @@ -225,7 +306,8 @@ int measureOffset; uint32_t flags; + int catID; } AverageTiny; -// a reduced-subset structure for relphot +// a reduced-subset structure for relphot & relastro typedef struct { float dR; @@ -242,5 +324,10 @@ unsigned int imageID; unsigned int dbFlags; + unsigned int photFlags; + int catID; // unsigned int? unsigned short photcode; + short dXccd; + short dYccd; + short dRsys; } MeasureTiny; @@ -383,9 +470,9 @@ int PhotColorTiny (AverageTiny *average, SecFilt *secfilt, MeasureTiny *measure, int c1, int c2, double *color); - PhotCodeData *GetPhotcodeTable (void); void SetPhotcodeTable (PhotCodeData *); int *GetSecFiltMap(PhotCodeData *ouput, PhotCodeData* input); +PhotCode **ParsePhotcodeList (char *rawlist, int *nphotcodes, int needAve); int LoadPhotcodes (char *catdir_file, char *master_file, int readwrite); @@ -450,4 +537,6 @@ int WriteRawSecFilt (FILE *f, SecFilt *secfilt, off_t Nsecfilt, char format); +DVOTableFormat FtableGetFormat (FTable *ftable); + Average *FtableToAverage (FTable *ftable, off_t *Naverage, char *format, SecFilt **primary); Measure *FtableToMeasure (FTable *ftable, off_t *Nmeasure, char *format); @@ -470,4 +559,5 @@ # include "ps1_v1_defs.h" # include "ps1_v2_defs.h" +# include "ps1_v3_defs.h" # include "ps1_ref_defs.h" @@ -484,7 +574,14 @@ void dvo_image_create (FITS_DB *db, double ZeroPoint); +/* flatcorr APIs */ +FlatCorrectionTable *FlatCorrectionLoad (char *filename, int VERBOSE); +int FlatCorrectionInternal(FlatCorrectionTable *flatcorrTable); +int FlatCorrectionSave (FlatCorrectionTable *flatcorrTable, char *filename); +float FlatCorrectionOffset (FlatCorrectionTable *flatcorr, int ID, int X, int Y); + /* skyregion APIs */ int SkyTableSave PROTO((SkyTable *table, char *filename)); SkyTable *SkyTableLoad PROTO((char *filename, int VERBOSE)); +char *SkyTableFilename PROTO((char *catdir)); SkyTable *SkyTableFromGSC PROTO((char *filename, int depth, int VERBOSE)); SkyTable *SkyTableLoadOptimal PROTO((char *catdir, char *SKYFILE, char *GSCFILE, int readwrite, int depth, int VERBOSE)); @@ -528,3 +625,16 @@ #endif -# endif +// functions for parallel DVO +HostTable *HostTableLoad (char *catdir, char *rootname); +int HostTableWaitJobs (HostTable *table, char *file, int lineno); +int HostTableWaitJobsGetIO (HostTable *table, char *file, int lineno, int VERBOSE); +int HostTableTestHost (SkyRegion *region, int hostID); + +// functions to support tiny versions of Average and Measure +void CopyAverageToTiny (AverageTiny *averageT, Average *average); +void CopyMeasureToTiny (MeasureTiny *measureT, Measure *measure); +int populate_tiny_values (Catalog *catalog, DVOTinyValueMode mode); +int free_tiny_values (Catalog *catalog); + +# endif // DVO_H + Index: trunk/Ohana/src/libdvo/include/dvodb.h =================================================================== --- trunk/Ohana/src/libdvo/include/dvodb.h (revision 33648) +++ trunk/Ohana/src/libdvo/include/dvodb.h (revision 33649) @@ -1,4 +1,6 @@ # ifndef DVODB_H # define DVODB_H + +# define MEASURE_HAS_XCCD 1 typedef enum { @@ -74,4 +76,5 @@ MEAS_NPOS, MEAS_OBJ_FLAGS, + MEAS_SECFILT_FLAGS, MEAS_MAG, MEAS_MINST, @@ -82,8 +85,11 @@ MEAS_EXPTIME, MEAS_AIRMASS, + MEAS_MEAN_AIRMASS, MEAS_ALT, MEAS_AZ, MEAS_PHOTCODE, MEAS_PHOTCODE_EQUIV, + MEAS_PHOTCODE_KLAM, + MEAS_PHOTCODE_C, MEAS_TIME, MEAS_FWHM, @@ -120,4 +126,6 @@ MEAS_EXT_NSIGMA, MEAS_EXTERN_ID, + MEAS_MCAL_OFFSET, + MEAS_FLAT, }; Index: trunk/Ohana/src/libdvo/include/ps1_v3_defs.h =================================================================== --- trunk/Ohana/src/libdvo/include/ps1_v3_defs.h (revision 33649) +++ trunk/Ohana/src/libdvo/include/ps1_v3_defs.h (revision 33649) @@ -0,0 +1,11 @@ +Image *Image_PS1_V3_ToInternal (Image_PS1_V3 *in, off_t Nvalues); +Image_PS1_V3 *ImageInternalTo_PS1_V3 (Image *in, off_t Nvalues); +Average *Average_PS1_V3_ToInternal (Average_PS1_V3 *in, off_t Nvalues, SecFilt **primary); +Average_PS1_V3 *AverageInternalTo_PS1_V3 (Average *in, off_t Nvalues, SecFilt *primary); +Measure *Measure_PS1_V3_ToInternal (Measure_PS1_V3 *in, off_t Nvalues); +Measure_PS1_V3 *MeasureInternalTo_PS1_V3 (Measure *in, off_t Nvalues); +SecFilt *SecFilt_PS1_V3_ToInternal (SecFilt_PS1_V3 *in, off_t Nvalues); +SecFilt_PS1_V3 *SecFiltInternalTo_PS1_V3 (SecFilt *in, off_t Nvalues); + +PhotCode *PhotCode_PS1_V3_To_Internal (PhotCode_PS1_V3 *in, off_t Nvalues); +PhotCode_PS1_V3 *PhotCode_Internal_To_PS1_V3 (PhotCode *in, off_t Nvalues); Index: trunk/Ohana/src/libdvo/src/HostTable.c =================================================================== --- trunk/Ohana/src/libdvo/src/HostTable.c (revision 33649) +++ trunk/Ohana/src/libdvo/src/HostTable.c (revision 33649) @@ -0,0 +1,385 @@ +# include "dvo.h" +# include +# include + +void InitHosts (HostInfo *hosts, int Nhosts, int NHOSTS) { + + int i; + for (i = Nhosts; i < NHOSTS; i++) { + hosts[i].hostname = NULL; + hosts[i].pathname = NULL; + hosts[i].results = NULL; + hosts[i].stdio[HOST_STDIN] = -1; + hosts[i].stdio[HOST_STDOUT] = -1; + hosts[i].stdio[HOST_STDERR] = -1; + hosts[i].pid = 0; + } + return; +} + +void FreeHosts (HostInfo *hosts, int Nhosts) { + + int i; + for (i = 0; i < Nhosts; i++) { + free (hosts[i].hostname); + free (hosts[i].pathname); + } + free (hosts); + return; +} + +void FreeTable (HostTable *table) { + + if (!table) return; + if (table->hosts) { + FreeHosts (table->hosts, table->Nhosts); + } + if (table->index) free (table->index); + free (table); + return; +} + +HostTable *HostTableLoad (char *catdir, char *rootname) { + + int i, Nline; + + char *filename = NULL; + + ALLOCATE (filename, char, strlen(catdir) + strlen(rootname) + 2); // one slash and one EOL + sprintf (filename, "%s/%s", catdir, rootname); + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "failed to open host table %s\n", filename); + free (filename); + return NULL; + } + + // simple format: ID hostname pathname + + int NHOSTS = 16; + int Nhosts = 0; + HostInfo *hosts = NULL; + ALLOCATE (hosts, HostInfo, NHOSTS); + InitHosts (hosts, Nhosts, NHOSTS); + + int maxID = 0; + + for (Nline = 0; TRUE; Nline ++) { + int ID; + char tmphost[1024]; + char tmppath[1024]; + char line[1024]; + + // XXXX use this for safety: int status = scan_line_maxlen (f, line, 1024); + int status = scan_line (f, line); + if (status == EOF) break; + + // find first non-whitespace char & skip commented lines + for (i = 0; OHANA_WHITESPACE (line[i]); i++); + if (line[i] == '#') continue; + if (line[i] == 0) continue; + + status = sscanf (line, "%d %1023s %1023s", &ID, tmphost, tmppath); + if (status != 3) { + fprintf (stderr, "error reading line %d of host table %s\n", Nline, filename); + FreeHosts (hosts, Nhosts); + free (filename); + return NULL; + } + + // check the validity of ID (0 < ID < MAX_SHORT) + + if (ID < 1) { + fprintf (stderr, "invalid host ID %d\n", ID); + exit (1); + } + if (ID > 255) { + fprintf (stderr, "invalid host ID %d\n", ID); + exit (1); + } + maxID = MAX(maxID, ID); + + hosts[Nhosts].hostID = ID; + hosts[Nhosts].hostname = strcreate(tmphost); + hosts[Nhosts].pathname = strcreate(tmppath); + + InitIOBuffer (&hosts[Nhosts].stdout, 1000); + InitIOBuffer (&hosts[Nhosts].stderr, 1000); + + Nhosts ++; + if (Nhosts >= NHOSTS) { + NHOSTS += 16; + REALLOCATE (hosts, HostInfo, NHOSTS); + InitHosts (hosts, Nhosts, NHOSTS); + } + } + + HostTable *table = NULL; + ALLOCATE (table, HostTable, 1); + table->Nhosts = Nhosts; + table->hosts = hosts; + + ALLOCATE (table->index, unsigned short, maxID + 1); + for (i = 0; i <= maxID; i++) table->index[i] = -1; + + for (i = 0; i < table->Nhosts; i++) { + if (table->index[table->hosts[i].hostID] == -1) { + fprintf (stderr, "error: duplicate hostID %d\n", table->hosts[i].hostID); + exit (1); + } + table->index[table->hosts[i].hostID] = i; + } + + free (filename); + return table; +} + +// wait for all children to complete, report output to stdout +int HostTableWaitJobs (HostTable *table, char *file, int lineno) { + + int i; + + // we have launched table->Nhosts jobs; wait for all of them to complete... + // if one (N) failed to launch, we will get an ECHILD error from the last (N) calls + int done = FALSE; + for (i = 0; !done && (i < table->Nhosts); i++) { + int status = 0; + + // XXX we'll need to pass in WNOHANG and keep retrying if we want to have a timeout... + int pid = waitpid (-1, &status, 0); + if (!pid) { + // this should only occur if we called waitpid with the WNOHANG option + fprintf (stderr, "programming error (1)? %s %d", file, lineno); + exit (2); + } + if (pid == -1) { + switch (errno) { + case ECHILD: + done = TRUE; + break; + default: + fprintf (stderr, "programming error (2)? %s %d", file, lineno); + exit (2); + } + } + + // when the host has finished, close the open sockets + + // find the host which has finished + int j; + int found = FALSE; + for (j = 0; j < table->Nhosts; j++) { + if (table->hosts[j].pid != pid) continue; + found = TRUE; + // check on the status of this and report any output? + fprintf (stderr, "job finished for %s (%d)\n", table->hosts[j].hostname, pid); + // read the stderr and stdout + IOBuffer buffer; + InitIOBuffer (&buffer, 100); + EmptyIOBuffer (&buffer, 100, table->hosts[j].stdio[HOST_STDOUT]); + fprintf (stderr, "--- stdout from %s ---\n", table->hosts[j].hostname); + write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer); + fprintf (stderr, "\n"); + + InitIOBuffer (&buffer, 100); + EmptyIOBuffer (&buffer, 100, table->hosts[j].stdio[HOST_STDERR]); + fprintf (stderr, "--- stderr from %s ---\n", table->hosts[j].hostname); + write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer); + fprintf (stderr, "\n"); + if (WIFEXITED(status)) { + fprintf (stderr, "normal completion, exit status is %d\n", WEXITSTATUS(status)); + table->hosts[j].status = WEXITSTATUS(status); + if (table->hosts[j].status) { + fprintf (stderr, "job failed on %s\n", table->hosts[j].hostname); + continue; + } + } else { + table->hosts[j].status = -1; + fprintf (stderr, "job exited abnormally on %s\n", table->hosts[j].hostname); + continue; + } + } + if (!found) { + fprintf (stderr, "Programming error: failed to matched finished job to known host!\n"); + exit (2); + } + } + return TRUE; +} + +// wait for all children to complete, report output to stdout +int HostTableWaitJobsGetIO (HostTable *table, char *file, int lineno, int VERBOSE) { + + // we have launched table->Nhosts jobs; wait for all of them to complete... + // if one (N) failed to launch, we will get an ECHILD error from the last (N) calls + + // we need to read any data waiting on stderr or stdout from these jobs, or the overfull + // buffers can cause a problem. we alternate between 'select' and 'waitpid' calls with + // timeouts for both + + // add all hosts' sockets to the fd_sets + fd_set rdSet, wtSet; + FD_ZERO (&rdSet); + FD_ZERO (&wtSet); + + // XXX can I set the fd_sets once, since I am not actually closing the fd's? + + int globalStatus = TRUE; + + int i; + int Nmax = 0; + for (i = 0; i < table->Nhosts; i++) { + if (!table->hosts[i].pid) continue; // any unconnected hosts should be skipped + FD_SET (table->hosts[i].stdio[HOST_STDIN], &wtSet); + Nmax = MAX (Nmax, table->hosts[i].stdio[HOST_STDIN]); + FD_SET (table->hosts[i].stdio[HOST_STDOUT], &rdSet); + Nmax = MAX (Nmax, table->hosts[i].stdio[HOST_STDOUT]); + FD_SET (table->hosts[i].stdio[HOST_STDERR], &rdSet); + Nmax = MAX (Nmax, table->hosts[i].stdio[HOST_STDERR]); + } + Nmax ++; + + int Nfound = 0; + + // this loop has 2 chunks: (a) check for I/O + (b) check for jobs done + while (1) { + + // Wait up to 0.5 second for host to provide I/O + // timeout gets mucked: need to reset before each select + struct timeval timeout; + timeout.tv_sec = 10; + timeout.tv_usec = 500000; + + int status = select (Nmax, NULL, &wtSet, NULL, &timeout); + if (status == -1) { + perror("select()"); + exit (2); + } + + // we have some sockets to check, check sockets for all hosts + for (i = 0; (status > 0) && (i < table->Nhosts); i++) { + if (FALSE && FD_ISSET (table->hosts[i].stdio[HOST_STDIN], &wtSet)) { + // this host is waiting for input : this is an error, so exit + fprintf (stderr, "host %s is waiting for input\n", table->hosts[i].hostname); + abort(); + } + + if (FD_ISSET (table->hosts[i].stdio[HOST_STDOUT], &rdSet)) { + // this host has waiting output : read to buffer, and dump if necessary + ReadtoIOBuffer (&table->hosts[i].stdout, table->hosts[i].stdio[HOST_STDOUT]); + // if (table->hosts[i].stdout.Nbuffer > 0x10000) { + if (table->hosts[i].stdout.Nbuffer > 0x1000) { + int printHead = VERBOSE || table->hosts[i].stdout.Nbuffer; + if (printHead) fprintf (stdout, "--- stdout from %s ---\n", table->hosts[i].hostname); + write (STDOUT_FILENO, table->hosts[i].stdout.buffer, table->hosts[i].stdout.Nbuffer); + FlushIOBuffer (&table->hosts[i].stdout); + if (printHead) fprintf (stdout, "\n"); + } + } + + if (FD_ISSET (table->hosts[i].stdio[HOST_STDERR], &rdSet)) { + // this host has waiting output : read to buffer, and dump if necessary + ReadtoIOBuffer (&table->hosts[i].stderr, table->hosts[i].stdio[HOST_STDERR]); + // if (table->hosts[i].stderr.Nbuffer > 0x10000) { + if (table->hosts[i].stderr.Nbuffer > 0x1000) { + int printHead = VERBOSE || table->hosts[i].stderr.Nbuffer; + if (printHead) fprintf (stdout, "--- stderr from %s ---\n", table->hosts[i].hostname); + write (STDOUT_FILENO, table->hosts[i].stderr.buffer, table->hosts[i].stderr.Nbuffer); + FlushIOBuffer (&table->hosts[i].stderr); + if (printHead) fprintf (stdout, "\n"); + } + } + } + + // now check if any children have finished... + while (TRUE) { + int status = 0; + int pid = waitpid (-1, &status, WNOHANG); + if (!pid) { + // fprintf (stderr, "no hosts to harvest\n"); + usleep (500000); + break; // no outstanding jobs have finished + } + if ((pid == -1) && (errno == ECHILD)) goto escape; // no more jobs on which to wait + if ((pid == -1) && (errno != ECHILD)) { + fprintf (stderr, "programming error (2)? %s %d", file, lineno); + exit (2); + } + + // find the host which has finished + int found = FALSE; + for (i = 0; (i < table->Nhosts) && !found; i++) { + if (table->hosts[i].pid != pid) continue; + found = TRUE; + + HostInfo *host = &table->hosts[i]; + + // check on the status of this and report any output? + if (VERBOSE) fprintf (stdout, "job finished for %s (%d)\n", host->hostname, pid); + + // read stdout + int printHead; + printHead = VERBOSE || host->stdout.Nbuffer; + EmptyIOBuffer (&host->stdout, 100, host->stdio[HOST_STDOUT]); + if (printHead) fprintf (stdout, "--- stdout from %s ---\n", host->hostname); + write (STDOUT_FILENO, host->stdout.buffer, host->stdout.Nbuffer); + FlushIOBuffer (&host->stdout); + if (printHead) fprintf (stdout, "\n"); + + // read stderr + printHead = VERBOSE || host->stderr.Nbuffer; + EmptyIOBuffer (&host->stderr, 100, host->stdio[HOST_STDERR]); + if (printHead) fprintf (stdout, "--- stderr from %s ---\n", host->hostname); + write (STDOUT_FILENO, host->stderr.buffer, host->stderr.Nbuffer); + FlushIOBuffer (&host->stderr); + if (printHead) fprintf (stdout, "\n"); + + if (WIFEXITED(status)) { + if (VERBOSE) fprintf (stdout, "normal completion, exit status is %d\n", WEXITSTATUS(status)); + host->status = WEXITSTATUS(status); + if (host->status) { + fprintf (stdout, "job failed on %s\n", host->hostname); + globalStatus = FALSE; + } + } else { + host->status = -1; + fprintf (stdout, "job exited abnormally on %s\n", host->hostname); + globalStatus = FALSE; + continue; + } + } + if (!found) { + fprintf (stderr, "Programming error: failed to matched finished job to known host!\n"); + exit (2); + } + Nfound ++; + if (Nfound == table->Nhosts) goto escape; // we've harvested all jobs + } + } + +escape: + + // close all opened connections + for (i = 0; i < table->Nhosts; i++) { + if (!table->hosts[i].pid) continue; // any unconnected hosts should be skipped + close (table->hosts[i].stdio[HOST_STDIN]); + close (table->hosts[i].stdio[HOST_STDOUT]); + close (table->hosts[i].stdio[HOST_STDERR]); + } + + return globalStatus; +} + +int HostTableTestHost (SkyRegion *region, int hostID) { + + // if hostID is not set, then we are not in a remote client + if (!hostID) return TRUE; + + if (region->hostFlags & DATA_USE_BCK) { + if (region->backupID == hostID) return TRUE; + } else { + if (region->hostID == hostID) return TRUE; + } + return FALSE; +} Index: trunk/Ohana/src/libdvo/src/LoadImages.c =================================================================== --- trunk/Ohana/src/libdvo/src/LoadImages.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/LoadImages.c (revision 33649) @@ -73,5 +73,5 @@ cacheNimage = *Nimage; imageCache = image; - lastFilename = strdup(filename); + lastFilename = strcreate(filename); lastModified = getLastModified(filename); } Index: trunk/Ohana/src/libdvo/src/LoadPhotcodesFITS.c =================================================================== --- trunk/Ohana/src/libdvo/src/LoadPhotcodesFITS.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/LoadPhotcodesFITS.c (revision 33649) @@ -41,83 +41,26 @@ gfits_scan (&db.theader, "EXTNAME", "%s", 1, extname); - if (!strcmp (extname, "DVO_PHOTCODE")) { - PhotCode_Elixir *photcode_elixir = gfits_table_get_PhotCode_Elixir (&db.ftable, &Ncode, &db.swapped); - if (!photcode_elixir) { - fprintf (stderr, "ERROR: failed to read photcodes\n"); - exit (2); - } - photcode = PhotCode_Elixir_To_Internal (photcode_elixir, Ncode); - free (photcode_elixir); - } +# define CONVERT_FORMAT(NAME, FORMAT) \ + if (!strcmp (extname, NAME)) { \ + PhotCode_##FORMAT *photcode_input = gfits_table_get_PhotCode_##FORMAT (&db.ftable, &Ncode, &db.swapped); \ + if (!photcode_input) { \ + fprintf (stderr, "ERROR: failed to read photcodes in LoadPhotcodesFITS.c\n"); \ + fprintf (stderr, "file is either corrupted or this is a programming error\n"); \ + fprintf (stderr, "exiting to avoid damaging dvo db\n"); \ + exit (2); \ + } \ + photcode = PhotCode_##FORMAT##_To_Internal (photcode_input, Ncode); \ + free (photcode_input); \ + } - if (!strcmp (extname, "DVO_PHOTCODE_ELIXIR")) { - PhotCode_Elixir *photcode_elixir = gfits_table_get_PhotCode_Elixir (&db.ftable, &Ncode, &db.swapped); - if (!photcode_elixir) { - fprintf (stderr, "ERROR: failed to read photcodes\n"); - exit (2); - } - photcode = PhotCode_Elixir_To_Internal (photcode_elixir, Ncode); - free (photcode_elixir); - } - - if (!strcmp (extname, "DVO_PHOTCODE_PS1_DEV_1")) { - PhotCode_PS1_DEV_1 *photcode_ps1_dev_1 = gfits_table_get_PhotCode_PS1_DEV_1 (&db.ftable, &Ncode, &db.swapped); - if (!photcode_ps1_dev_1) { - fprintf (stderr, "ERROR: failed to read photcodes\n"); - exit (2); - } - photcode = PhotCode_PS1_DEV_1_To_Internal (photcode_ps1_dev_1, Ncode); - free (photcode_ps1_dev_1); - } - - if (!strcmp (extname, "DVO_PHOTCODE_PS1_DEV_2")) { - PhotCode_PS1_DEV_2 *photcode_ps1_dev_2 = gfits_table_get_PhotCode_PS1_DEV_2 (&db.ftable, &Ncode, &db.swapped); - if (!photcode_ps1_dev_2) { - fprintf (stderr, "ERROR: failed to read photcodes\n"); - exit (2); - } - photcode = PhotCode_PS1_DEV_2_To_Internal (photcode_ps1_dev_2, Ncode); - free (photcode_ps1_dev_2); - } - - if (!strcmp (extname, "DVO_PHOTCODE_PS1_DEV_3")) { - PhotCode_PS1_DEV_3 *photcode_ps1_dev_3 = gfits_table_get_PhotCode_PS1_DEV_3 (&db.ftable, &Ncode, &db.swapped); - if (!photcode_ps1_dev_3) { - fprintf (stderr, "ERROR: failed to read photcodes\n"); - exit (2); - } - photcode = PhotCode_PS1_DEV_3_To_Internal (photcode_ps1_dev_3, Ncode); - free (photcode_ps1_dev_3); - } - - if (!strcmp (extname, "DVO_PHOTCODE_PS1_V1")) { - PhotCode_PS1_V1 *photcode_ps1_v1 = gfits_table_get_PhotCode_PS1_V1 (&db.ftable, &Ncode, &db.swapped); - if (!photcode_ps1_v1) { - fprintf (stderr, "ERROR: failed to read photcodes\n"); - exit (2); - } - photcode = PhotCode_PS1_V1_To_Internal (photcode_ps1_v1, Ncode); - free (photcode_ps1_v1); - } - - if (!strcmp (extname, "DVO_PHOTCODE_PS1_V2")) { - PhotCode_PS1_V2 *photcode_ps1_v2 = gfits_table_get_PhotCode_PS1_V2 (&db.ftable, &Ncode, &db.swapped); - if (!photcode_ps1_v2) { - fprintf (stderr, "ERROR: failed to read photcodes\n"); - exit (2); - } - photcode = PhotCode_PS1_V2_To_Internal (photcode_ps1_v2, Ncode); - free (photcode_ps1_v2); - } - - if (!strcmp (extname, "DVO_PHOTCODE_PS1_REF")) { - PhotCode_PS1_REF *photcode_ps1_ref = gfits_table_get_PhotCode_PS1_REF (&db.ftable, &Ncode, &db.swapped); - if (!photcode_ps1_ref) { - fprintf (stderr, "ERROR: failed to read photcodes\n"); - exit (2); - } - photcode = PhotCode_PS1_REF_To_Internal (photcode_ps1_ref, Ncode); - free (photcode_ps1_ref); - } + CONVERT_FORMAT("DVO_PHOTCODE", Elixir); + CONVERT_FORMAT("DVO_PHOTCODE_ELIXIR", Elixir); + CONVERT_FORMAT("DVO_PHOTCODE_PS1_DEV_1", PS1_DEV_1); + CONVERT_FORMAT("DVO_PHOTCODE_PS1_DEV_2", PS1_DEV_2); + CONVERT_FORMAT("DVO_PHOTCODE_PS1_DEV_3", PS1_DEV_3); + CONVERT_FORMAT("DVO_PHOTCODE_PS1_REF", PS1_REF); + CONVERT_FORMAT("DVO_PHOTCODE_PS1_V1", PS1_V1); + CONVERT_FORMAT("DVO_PHOTCODE_PS1_V2", PS1_V2); + CONVERT_FORMAT("DVO_PHOTCODE_PS1_V3", PS1_V3); table = GetPhotcodeTable (); Index: trunk/Ohana/src/libdvo/src/SavePhotcodesFITS.c =================================================================== --- trunk/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 33649) @@ -29,9 +29,9 @@ // for the moment, we simply support the latest photcode format for output // XXX update this as needed as new formats are defined - PhotCode_PS1_V2 *photcode_output = PhotCode_Internal_To_PS1_V2 (table[0].code, table[0].Ncode); + PhotCode_PS1_V3 *photcode_output = PhotCode_Internal_To_PS1_V3 (table[0].code, table[0].Ncode); /* convert FITS format data to internal format (byteswaps & EXTNAME) */ if (!gfits_db_create (&db)) return (FALSE); - if (!gfits_table_set_PhotCode_PS1_V2 (&db.ftable, photcode_output, table[0].Ncode)) return (FALSE); + if (!gfits_table_set_PhotCode_PS1_V3 (&db.ftable, photcode_output, table[0].Ncode)) return (FALSE); if (!gfits_db_save (&db)) return (FALSE); if (!gfits_db_close (&db)) return (FALSE); Index: trunk/Ohana/src/libdvo/src/dbExtractAverages.c =================================================================== --- trunk/Ohana/src/libdvo/src/dbExtractAverages.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dbExtractAverages.c (revision 33649) @@ -208,5 +208,7 @@ } } else { - value.Int = 0; + Nsec = GetPhotcodeNsec (field->photcode->code); + if (Nsec == -1) break; + value.Int = secfilt[Nsec].flags; } break; Index: trunk/Ohana/src/libdvo/src/dbExtractMeasures.c =================================================================== --- trunk/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 33649) @@ -277,4 +277,11 @@ value.Int = average[0].flags; break; + case MEAS_SECFILT_FLAGS: /* OK */ + equiv = GetPhotcodeEquivbyCode (measure[0].photcode); + Nsec = GetPhotcodeNsec (equiv->code); + if (Nsec == -1) break; + value.Int = secfilt[Nsec].flags; + break; + // note that these represent the ra displacement relative to the average, not // the error. @@ -302,4 +309,11 @@ value.Flt = measure[0].airmass; break; + case MEAS_MEAN_AIRMASS: /* OK */ + { + Image *image = MatchImageDVO (measure[0].t, measure[0].photcode, measure[0].imageID); + if (image == NULL) break; + value.Flt = image[0].secz; + break; + } case MEAS_AZ: /* OK */ value.Flt = measure[0].az; @@ -317,4 +331,18 @@ value.Int = measure[0].photcode; break; + case MEAS_PHOTCODE_KLAM: /* OK */ + { + PhotCode *code = GetPhotcodebyCode (measure[0].photcode); + if (!code) break; + value.Flt = code->K; + break; + } + case MEAS_PHOTCODE_C: /* OK */ + { + PhotCode *code = GetPhotcodebyCode (measure[0].photcode); + if (!code) break; + value.Flt = code->C; + break; + } case MEAS_TIME: /* OK */ value.Flt = TimeValue (measure[0].t, TimeReference, TimeFormat); @@ -351,9 +379,10 @@ break; case MEAS_XCCD: /* OK */ -/* I need to perform this conversion for ELIXIR and LONEOS formats on load */ -# if 1 +# if MEASURE_HAS_XCCD value.Flt = measure[0].Xccd; # else { + // I'm keeping this code because it gives a way of handing dvo dbs that don't have + // measure.xccd if we need it Image *image; ra = average[0].R - measure[0].dR / 3600.0; @@ -367,6 +396,5 @@ break; case MEAS_YCCD: /* OK */ -/* I need to perform this conversion for ELIXIR and LONEOS formats on load */ -# if 1 +# if MEASURE_HAS_XCCD value.Flt = measure[0].Yccd; # else @@ -456,4 +484,18 @@ break; + case MEAS_MCAL_OFFSET: /* OK */ + value.Flt = measure[0].Mcal; + break; + + case MEAS_FLAT: /* OK */ + // flat = measure.Mcal - image.Mcal + { + Image *image; + image = MatchImageDVO (measure[0].t, measure[0].photcode, measure[0].imageID); + if (image == NULL) break; + value.Flt = measure[0].Mcal - image[0].Mcal; + } + break; + // add the star/galaxy sep -- } Index: trunk/Ohana/src/libdvo/src/dbFields.c =================================================================== --- trunk/Ohana/src/libdvo/src/dbFields.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dbFields.c (revision 33649) @@ -42,4 +42,5 @@ if (!strcasecmp (string, "aveerr")) return (MAG_AVE_ERR); if (!strcasecmp (string, "photflags")) return (MAG_PHOT_FLAGS); + if (!strcasecmp (string, "flags")) return (MAG_PHOT_FLAGS); if (!strcasecmp (string, "chisq")) return (MAG_CHISQ); if (!strcasecmp (string, "ncode")) return (MAG_NCODE); @@ -169,4 +170,8 @@ if (!strcasecmp (fieldName, "OBJFLAGS")) ESCAPE (MEAS_OBJ_FLAGS, MAG_NONE, OPIHI_INT); if (!strcasecmp (fieldName, "OBJ_FLAGS")) ESCAPE (MEAS_OBJ_FLAGS, MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "OBJ_PHOT_FLAGS")) ESCAPE (MEAS_SECFILT_FLAGS, MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "SECFLAGS")) ESCAPE (MEAS_SECFILT_FLAGS, MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "SEC_FLAGS")) ESCAPE (MEAS_SECFILT_FLAGS, MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "SECFILT_FLAGS")) ESCAPE (MEAS_SECFILT_FLAGS, MAG_NONE, OPIHI_INT); if (!strcasecmp (fieldName, "DB_FLAGS")) ESCAPE (MEAS_DB_FLAGS, MAG_NONE, OPIHI_INT); if (!strcasecmp (fieldName, "PHOT_FLAGS")) ESCAPE (MEAS_PHOT_FLAGS, MAG_NONE, OPIHI_INT); @@ -174,8 +179,11 @@ if (!strcasecmp (fieldName, "PHOTFLAGS")) ESCAPE (MEAS_PHOT_FLAGS, MAG_NONE, OPIHI_INT); if (!strcasecmp (fieldName, "AIRMASS")) ESCAPE (MEAS_AIRMASS, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "MEAN_AIRMASS")) ESCAPE (MEAS_MEAN_AIRMASS, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "ALT")) ESCAPE (MEAS_ALT, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "AZ")) ESCAPE (MEAS_AZ, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "EXPTIME")) ESCAPE (MEAS_EXPTIME, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "PHOTCODE")) ESCAPE (MEAS_PHOTCODE, MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "PHOTCODE:KLAM")) ESCAPE (MEAS_PHOTCODE_KLAM, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "PHOTCODE:C")) ESCAPE (MEAS_PHOTCODE_C, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "PHOTCODE:EQUIV")) ESCAPE (MEAS_PHOTCODE_EQUIV, MAG_NONE, OPIHI_INT); if (!strcasecmp (fieldName, "TIME")) ESCAPE (MEAS_TIME, MAG_NONE, OPIHI_FLT); @@ -215,4 +223,6 @@ if (!strcasecmp (fieldName, "SKY_ERR")) ESCAPE (MEAS_dSKY, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "externID")) ESCAPE (MEAS_EXTERN_ID, MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "MCAL_OFFSET")) ESCAPE (MEAS_MCAL_OFFSET, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "FLAT")) ESCAPE (MEAS_FLAT, MAG_NONE, OPIHI_FLT); // for words that don't parse, try a photcode @@ -286,5 +296,5 @@ if (!strcasecmp (fieldName, "dPAR")) ESCAPE (AVE_PAR_ERR, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "ChiSqPos")) ESCAPE (AVE_CHISQ_POS, MAG_NONE, OPIHI_FLT); - if (!strcasecmp (fieldName, "ChiSqPM")) ESCAPE (AVE_CHISQ_PM, MAG_NONE, OPIHI_FLT); + if (!strcasecmp (fieldName, "ChiSqPM")) ESCAPE (AVE_CHISQ_PM, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "ChiSqPar")) ESCAPE (AVE_CHISQ_PAR, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "Tmean")) ESCAPE (AVE_TMEAN, MAG_NONE, OPIHI_FLT); Index: trunk/Ohana/src/libdvo/src/dvo_catalog.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_catalog.c (revision 33649) @@ -54,4 +54,5 @@ if (!strcasecmp (catformat, "PS1_V1")) return (DVO_FORMAT_PS1_V1); if (!strcasecmp (catformat, "PS1_V2")) return (DVO_FORMAT_PS1_V2); + if (!strcasecmp (catformat, "PS1_V3")) return (DVO_FORMAT_PS1_V3); if (!strcasecmp (catformat, "PS1_REF")) return (DVO_FORMAT_PS1_REF); return (DVO_FORMAT_UNDEF); @@ -250,4 +251,7 @@ dvo_catalog_init (catalog, FALSE); + + // reset the file pointer in case we've already read some data + fseeko (catalog[0].f, 0, SEEK_SET); // load the main catalog header, determine characteristics Index: trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 33649) @@ -3,24 +3,18 @@ int dvo_catalog_load_mef (Catalog *catalog, int VERBOSE) { + off_t Nbytes; + off_t Naverage; + off_t Nmeasure; + off_t Nmissing; + off_t Nitems, Nexpect; int Nsecfilt; - off_t Nitems, Nbytes, Nexpect, Naverage, Nmeasure, Nmissing; - FILE *f; Header header; - Matrix matrix; FTable ftable; - SecFilt *primary; - - f = catalog[0].f; + + SecFilt *primary = NULL; + ftable.header = &header; - /* move pointer past header -- must be already read (load_catalog) */ - fseeko (f, catalog[0].header.datasize, SEEK_SET); - - /* matrix should be empty */ - if (!gfits_fread_matrix (f, &matrix, &catalog[0].header)) { - if (VERBOSE) fprintf (stderr, "can't read primary matrix"); - return (FALSE); - } /* get the components from the header */ if (!gfits_scan (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, &Naverage)) return (FALSE); @@ -57,14 +51,22 @@ catalog[0].secfilt = NULL; - /* validate table mode */ + /* validate table mode ?*/ + + /*** Average Table ***/ + + /* need to read the table header to determine format (whether or not we read averages) */ + /* move pointer past PHU header -- must be already read (load_catalog) */ + Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header); + fseeko (catalog[0].f, Nbytes, SEEK_SET); /* read Average table header */ - if (!gfits_fread_header (f, &header)) { + if (!gfits_fread_header (catalog[0].f, &header)) { if (VERBOSE) fprintf (stderr, "can't read table average header"); return (FALSE); } + /* read Average table data (or skip) */ if (catalog[0].catflags & LOAD_AVES) { - if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table average data"); return (FALSE); @@ -78,6 +80,7 @@ catalog[0].Naves_off = 0; } else { + catalog[0].catformat = FtableGetFormat (&ftable); Nbytes = gfits_data_size (&header); - fseeko (f, Nbytes, SEEK_CUR); + fseeko (catalog[0].f, Nbytes, SEEK_CUR); ALLOCATE (catalog[0].average, Average, 1); catalog[0].Naverage = 0; @@ -88,5 +91,5 @@ /* read Measure table header */ - if (!gfits_fread_header (f, &header)) { + if (!gfits_fread_header (catalog[0].f, &header)) { if (VERBOSE) fprintf (stderr, "can't read table measure header"); return (FALSE); @@ -94,5 +97,5 @@ /* read Measure table data */ if (catalog[0].catflags & LOAD_MEAS) { - if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table measure data"); return (FALSE); @@ -106,5 +109,5 @@ } else { Nbytes = gfits_data_size (&header); - fseeko (f, Nbytes, SEEK_CUR); + fseeko (catalog[0].f, Nbytes, SEEK_CUR); ALLOCATE (catalog[0].measure, Measure, 1); catalog[0].Nmeasure = 0; @@ -113,5 +116,5 @@ /* read Missing table header */ - if (!gfits_fread_header (f, &header)) { + if (!gfits_fread_header (catalog[0].f, &header)) { if (VERBOSE) fprintf (stderr, "can't read table missing header"); return (FALSE); @@ -119,5 +122,5 @@ /* read Missing table data */ if (catalog[0].catflags & LOAD_MISS) { - if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table missing data"); return (FALSE); @@ -136,5 +139,5 @@ } else { Nbytes = gfits_data_size (&header); - fseeko (f, Nbytes, SEEK_CUR); + fseeko (catalog[0].f, Nbytes, SEEK_CUR); ALLOCATE (catalog[0].missing, Missing, 1); catalog[0].Nmissing = 0; @@ -143,5 +146,5 @@ /* read secfilt table header */ - if (!gfits_fread_header (f, &header)) { + if (!gfits_fread_header (catalog[0].f, &header)) { if (VERBOSE) fprintf (stderr, "can't read table secfilt header"); return (FALSE); @@ -149,5 +152,5 @@ /* read secfilt table data */ if (catalog[0].catflags & LOAD_SECF) { - if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table secfilt data"); return (FALSE); @@ -187,5 +190,5 @@ /* no real need to skip the data array here... */ Nbytes = gfits_data_size (&header); - fseeko (f, Nbytes, SEEK_CUR); + fseeko (catalog[0].f, Nbytes, SEEK_CUR); if (primary != NULL) { free (primary); Index: trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 33649) @@ -88,4 +88,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); @@ -280,4 +281,5 @@ if (catalog[0].catformat == DVO_FORMAT_PS1_V1) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_V1"); if (catalog[0].catformat == DVO_FORMAT_PS1_V2) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_V2"); + if (catalog[0].catformat == DVO_FORMAT_PS1_V3) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_V3"); if (catalog[0].catformat == DVO_FORMAT_PS1_REF) gfits_modify (&catalog[0].header, "FORMAT", "%s", 1, "PS1_REF"); @@ -381,4 +383,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); @@ -430,4 +433,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); @@ -483,4 +487,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); @@ -532,4 +537,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); @@ -585,4 +591,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); @@ -634,4 +641,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); Index: trunk/Ohana/src/libdvo/src/dvo_catalog_split.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 33649) @@ -160,5 +160,7 @@ off_t Nbytes; - off_t Naverage, Nmeasure, Nmissing; + off_t Naverage; + off_t Nmeasure; + off_t Nmissing; off_t Nitems; int status, Nsecfilt; @@ -209,13 +211,16 @@ /*** Average Table ***/ + + /* need to read the table header to determine format (whether or not we read averages) */ + /* move pointer past PHU header -- must be already read (load_catalog) */ + Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header); + fseeko (catalog[0].f, Nbytes, SEEK_SET); + + /* read Average table header */ + if (!gfits_fread_header (catalog[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table average header"); + return (FALSE); + } if (catalog[0].catflags & LOAD_AVES) { - /* move pointer past header -- must be already read (load_catalog) */ - Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header); - fseeko (catalog[0].f, Nbytes, SEEK_SET); - /* read Average table header */ - if (!gfits_fread_header (catalog[0].f, &header)) { - if (VERBOSE) fprintf (stderr, "can't read table average header"); - return (FALSE); - } /* read Average table data : format is irrelevant here */ if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { @@ -234,4 +239,5 @@ catalog[0].Naves_off = 0; } else { + catalog[0].catformat = FtableGetFormat (&ftable); ALLOCATE (catalog[0].average, Average, 1); catalog[0].Naverage = 0; Index: trunk/Ohana/src/libdvo/src/dvo_convert.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert.c (revision 33649) @@ -10,4 +10,40 @@ /** this file might be more readable if I use macros for the repetative constructions below **/ + +DVOTableFormat FtableGetFormat (FTable *ftable) { + + DVOTableFormat format; + char extname[80]; + + /* read the format of this table */ + if (!gfits_scan (ftable[0].header, "EXTNAME", "%s", 1, extname)) { + fprintf (stderr, "EXTNAME missing for average table\n"); + return (DVO_FORMAT_UNDEF); + } + + if (!strcmp (extname, "DVO_AVERAGE")) { + format = DVO_FORMAT_INTERNAL; + return (format); + } +# define CONVERT_FORMAT(NAME, FORMAT) \ + if (!strcmp (extname, NAME)) { \ + format = DVO_FORMAT_##FORMAT; \ + return (format); } + + CONVERT_FORMAT ("DVO_AVERAGE_ELIXIR", ELIXIR); + CONVERT_FORMAT ("DVO_AVERAGE_LONEOS", LONEOS); + CONVERT_FORMAT ("DVO_AVERAGE_PANSTARRS_DEV_0", PANSTARRS_DEV_0); + CONVERT_FORMAT ("DVO_AVERAGE_PANSTARRS_DEV_1", PANSTARRS_DEV_1); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_DEV_1", PS1_DEV_1); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_DEV_2", PS1_DEV_2); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_V1", PS1_V1); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_V2", PS1_V2); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_V3", PS1_V3); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_REF", PS1_REF); +# undef CONVERT_FORMAT + + fprintf (stderr, "table format unknown: %s\n", extname); + return (DVO_FORMAT_UNDEF); +} /*** Average / FTable conversion functions ***/ @@ -59,4 +95,5 @@ CONVERT_FORMAT ("DVO_AVERAGE_PS1_V1", PS1_V1, PS1_V1); CONVERT_FORMAT ("DVO_AVERAGE_PS1_V2", PS1_V2, PS1_V2); + CONVERT_FORMAT ("DVO_AVERAGE_PS1_V3", PS1_V3, PS1_V3); CONVERT_FORMAT ("DVO_AVERAGE_PS1_REF", PS1_REF, PS1_REF); # undef CONVERT_FORMAT @@ -92,4 +129,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); # undef FORMAT_CASE @@ -146,4 +184,5 @@ CONVERT_FORMAT ("DVO_MEASURE_PS1_V1", PS1_V1, PS1_V1); CONVERT_FORMAT ("DVO_MEASURE_PS1_V2", PS1_V2, PS1_V2); + CONVERT_FORMAT ("DVO_MEASURE_PS1_V3", PS1_V3, PS1_V3); CONVERT_FORMAT ("DVO_MEASURE_PS1_REF", PS1_REF, PS1_REF); # undef CONVERT_FORMAT @@ -179,4 +218,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); # undef FORMAT_CASE @@ -233,4 +273,5 @@ CONVERT_FORMAT ("DVO_SECFILT_PS1_V1", PS1_V1, PS1_V1); CONVERT_FORMAT ("DVO_SECFILT_PS1_V2", PS1_V2, PS1_V2); + CONVERT_FORMAT ("DVO_SECFILT_PS1_V3", PS1_V3, PS1_V3); CONVERT_FORMAT ("DVO_SECFILT_PS1_REF", PS1_REF, PS1_REF); # undef CONVERT_FORMAT @@ -266,4 +307,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); # undef FORMAT_CASE @@ -344,4 +386,5 @@ CONVERT_FORMAT ("DVO_IMAGE_PS1_V1", PS1_V1, PS1_V1); CONVERT_FORMAT ("DVO_IMAGE_PS1_V2", PS1_V2, PS1_V2); + CONVERT_FORMAT ("DVO_IMAGE_PS1_V3", PS1_V3, PS1_V3); CONVERT_FORMAT ("DVO_IMAGE_PS1_REF", PS1_REF, PS1_REF); @@ -381,4 +424,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); @@ -434,4 +478,5 @@ FORMAT_CASE (PS1_V1, PS1_V1); FORMAT_CASE (PS1_V2, PS1_V2); + FORMAT_CASE (PS1_V3, PS1_V3); FORMAT_CASE (PS1_REF, PS1_REF); Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c (revision 33649) @@ -147,4 +147,8 @@ out[i].Npos = 0.0; out[i].extID = 0; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -206,4 +210,8 @@ // changed or added for PS1_V2 out[i].flags = 0; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -268,5 +276,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -335,5 +343,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 33649) @@ -141,4 +141,8 @@ out[i].Npos = 0.0; out[i].extID = 0; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -200,4 +204,8 @@ // changed or added for PS1_V2 out[i].flags = 0; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -264,5 +272,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -329,5 +337,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_3.c (revision 33649) @@ -48,5 +48,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -107,5 +107,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c (revision 33649) @@ -114,4 +114,8 @@ out[i].catID = in[i].catID; out[i].extID = 0; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -160,4 +164,8 @@ // changed or added for PS1_V2 out[i].flags = 0; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -223,5 +231,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -285,5 +293,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 33649) @@ -146,4 +146,8 @@ out[i].catID = in[i].catID; out[i].extID = in[i].extID; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -204,4 +208,8 @@ // changed or added for PS1_V2 out[i].flags = 0; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -272,5 +280,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -336,5 +344,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 33649) @@ -147,4 +147,8 @@ out[i].catID = in[i].catID; out[i].extID = in[i].extID; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -207,4 +211,8 @@ out[i].M_20 = in[i].M_20; out[i].M_80 = in[i].M_80; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -281,5 +289,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -343,5 +351,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 33649) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 33649) @@ -0,0 +1,435 @@ +# include + +/* convert PS1_V3 formats to internal formats */ + +Measure *Measure_PS1_V3_ToInternal (Measure_PS1_V3 *in, off_t Nvalues) { + + off_t i; + Measure *out; + + ALLOCATE_ZERO (out, Measure, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].dR = in[i].dR; + out[i].dD = in[i].dD; + out[i].M = in[i].M; + out[i].Mcal = in[i].Mcal; + out[i].Map = in[i].Map; + out[i].dM = in[i].dM; + out[i].dMcal = in[i].dMcal; + out[i].dt = in[i].dt; + out[i].airmass = in[i].airmass; + out[i].az = in[i].az; + out[i].Xccd = in[i].Xccd; + out[i].Yccd = in[i].Yccd; + out[i].Sky = in[i].Sky; + out[i].dSky = in[i].dSky; + out[i].t = in[i].t; + out[i].t_msec = in[i].t_msec; + out[i].averef = in[i].averef; + out[i].detID = in[i].detID; + out[i].imageID = in[i].imageID; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].extID = in[i].extID; + out[i].psfQual = in[i].psfQual; + out[i].psfChisq = in[i].psfChisq; + out[i].psfNdof = in[i].psfNdof; + out[i].psfNpix = in[i].psfNpix; + out[i].crNsigma = in[i].crNsigma; + out[i].extNsigma = in[i].extNsigma; + out[i].FWx = in[i].FWx; + out[i].FWy = in[i].FWy; + out[i].theta = in[i].theta; + out[i].Mxx = in[i].Mxx; + out[i].Mxy = in[i].Mxy; + out[i].Myy = in[i].Myy; + out[i].dXccd = in[i].dXccd; + out[i].dYccd = in[i].dYccd; + out[i].dRsys = in[i].dRsys; + out[i].posangle = in[i].posangle; + out[i].pltscale = in[i].pltscale; + out[i].photcode = in[i].photcode; + out[i].dbFlags = in[i].dbFlags; + out[i].photFlags = in[i].photFlags; + } + return (out); +} + +Measure_PS1_V3 *MeasureInternalTo_PS1_V3 (Measure *in, off_t Nvalues) { + + off_t i; + Measure_PS1_V3 *out; + + ALLOCATE_ZERO (out, Measure_PS1_V3, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].dR = in[i].dR; + out[i].dD = in[i].dD; + out[i].M = in[i].M; + out[i].Mcal = in[i].Mcal; + out[i].Map = in[i].Map; + out[i].dM = in[i].dM; + out[i].dMcal = in[i].dMcal; + out[i].dt = in[i].dt; + out[i].airmass = in[i].airmass; + out[i].az = in[i].az; + out[i].Xccd = in[i].Xccd; + out[i].Yccd = in[i].Yccd; + out[i].Sky = in[i].Sky; + out[i].dSky = in[i].dSky; + out[i].t = in[i].t; + out[i].t_msec = in[i].t_msec; + out[i].averef = in[i].averef; + out[i].detID = in[i].detID; + out[i].imageID = in[i].imageID; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].extID = in[i].extID; + out[i].psfQual = in[i].psfQual; + out[i].psfChisq = in[i].psfChisq; + out[i].psfNdof = in[i].psfNdof; + out[i].psfNpix = in[i].psfNpix; + out[i].crNsigma = in[i].crNsigma; + out[i].extNsigma = in[i].extNsigma; + out[i].FWx = in[i].FWx; + out[i].FWy = in[i].FWy; + out[i].theta = in[i].theta; + out[i].Mxx = in[i].Mxx; + out[i].Mxy = in[i].Mxy; + out[i].Myy = in[i].Myy; + out[i].dXccd = in[i].dXccd; + out[i].dYccd = in[i].dYccd; + out[i].dRsys = in[i].dRsys; + out[i].posangle = in[i].posangle; + out[i].pltscale = in[i].pltscale; + out[i].photcode = in[i].photcode; + out[i].dbFlags = in[i].dbFlags; + out[i].photFlags = in[i].photFlags; + } + return (out); +} + +// 'primary' is needed to conform with the API for Loneos and Elixir, but is not used +Average *Average_PS1_V3_ToInternal (Average_PS1_V3 *in, off_t Nvalues, SecFilt **primary) { + + off_t i; + Average *out; + + ALLOCATE_ZERO (out, Average, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].R = in[i].R; + out[i].D = in[i].D; + out[i].dR = in[i].dR; + out[i].dD = in[i].dD; + out[i].uR = in[i].uR; + out[i].uD = in[i].uD; + out[i].duR = in[i].duR; + out[i].duD = in[i].duD; + out[i].P = in[i].P; + out[i].dP = in[i].dP; + out[i].Xp = in[i].Xp; + out[i].ChiSqAve = in[i].ChiSqAve; + out[i].ChiSqPM = in[i].ChiSqPM; + out[i].ChiSqPar = in[i].ChiSqPar; + out[i].Tmean = in[i].Tmean; + out[i].Trange = in[i].Trange; + out[i].Npos = in[i].Npos; + out[i].Nmeasure = in[i].Nmeasure; + out[i].Nmissing = in[i].Nmissing; + out[i].Nextend = in[i].Nextend; + out[i].measureOffset = in[i].measureOffset; + out[i].missingOffset = in[i].missingOffset; + out[i].extendOffset = in[i].extendOffset; + out[i].flags = in[i].flags; + out[i].photFlagsUpper = in[i].photFlagsUpper; + out[i].photFlagsLower = in[i].photFlagsLower; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].extID = in[i].extID; + } + return (out); +} + +// 'primary' is needed to conform with the API for Loneos and Elixir, but is not used +Average_PS1_V3 *AverageInternalTo_PS1_V3 (Average *in, off_t Nvalues, SecFilt *primary) { + + off_t i; + Average_PS1_V3 *out; + + ALLOCATE_ZERO (out, Average_PS1_V3, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].R = in[i].R; + out[i].D = in[i].D; + out[i].dR = in[i].dR; + out[i].dD = in[i].dD; + out[i].uR = in[i].uR; + out[i].uD = in[i].uD; + out[i].duR = in[i].duR; + out[i].duD = in[i].duD; + out[i].P = in[i].P; + out[i].dP = in[i].dP; + out[i].Xp = in[i].Xp; + out[i].ChiSqAve = in[i].ChiSqAve; + out[i].ChiSqPM = in[i].ChiSqPM; + out[i].ChiSqPar = in[i].ChiSqPar; + out[i].Tmean = in[i].Tmean; + out[i].Trange = in[i].Trange; + out[i].Npos = in[i].Npos; + out[i].Nmeasure = in[i].Nmeasure; + out[i].Nmissing = in[i].Nmissing; + out[i].Nextend = in[i].Nextend; + out[i].measureOffset = in[i].measureOffset; + out[i].missingOffset = in[i].missingOffset; + out[i].extendOffset = in[i].extendOffset; + out[i].flags = in[i].flags; + out[i].photFlagsUpper = in[i].photFlagsUpper; + out[i].photFlagsLower = in[i].photFlagsLower; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].extID = in[i].extID; + } + return (out); +} + +SecFilt *SecFilt_PS1_V3_ToInternal (SecFilt_PS1_V3 *in, off_t Nvalues) { + + off_t i; + SecFilt *out; + + ALLOCATE_ZERO (out, SecFilt, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].M = in[i].M; + out[i].Map = in[i].Map; + out[i].dM = in[i].dM; + out[i].Xm = in[i].Xm; + out[i].flags = in[i].flags; + out[i].Ncode = in[i].Ncode; + out[i].Nused = in[i].Nused; + out[i].M_20 = in[i].M_20; + out[i].M_80 = in[i].M_80; + out[i].ubercalDist = in[i].ubercalDist; + } + return (out); +} + +SecFilt_PS1_V3 *SecFiltInternalTo_PS1_V3 (SecFilt *in, off_t Nvalues) { + + off_t i; + SecFilt_PS1_V3 *out; + + ALLOCATE_ZERO (out, SecFilt_PS1_V3, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].M = in[i].M; + out[i].Map = in[i].Map; + out[i].dM = in[i].dM; + out[i].Xm = in[i].Xm; + out[i].flags = in[i].flags; + out[i].Ncode = in[i].Ncode; + out[i].Nused = in[i].Nused; + out[i].M_20 = in[i].M_20; + out[i].M_80 = in[i].M_80; + out[i].ubercalDist = in[i].ubercalDist; + } + return (out); +} + +Image *Image_PS1_V3_ToInternal (Image_PS1_V3 *in, off_t Nvalues) { + + off_t i; + Image *out; + + ALLOCATE_ZERO (out, Image, Nvalues); + + for (i = 0; i < Nvalues; i++) { + memcpy (&out[i].coords, &in[i].coords, sizeof(Coords)); + + strncpy (out[i].name, in[i].name, 120); // out[121], in[121] + out[i].name[120] = 0; // force termination + + out[i].tzero = in[i].tzero; + out[i].nstar = in[i].nstar; + out[i].secz = in[i].secz; + out[i].NX = in[i].NX; + out[i].NY = in[i].NY; + out[i].apmifit = in[i].apmifit; + out[i].dapmifit = in[i].dapmifit; + out[i].Mcal = in[i].Mcal; + out[i].dMcal = in[i].dMcal; + out[i].Xm = in[i].Xm; + out[i].photcode = in[i].photcode; + out[i].exptime = in[i].exptime; + out[i].sidtime = in[i].sidtime; + out[i].latitude = in[i].latitude; + + out[i].RAo = in[i].RAo; + out[i].DECo = in[i].DECo; + out[i].Radius = in[i].Radius; + + out[i].detection_limit = in[i].detection_limit; + out[i].saturation_limit = in[i].saturation_limit; + out[i].cerror = in[i].cerror; + out[i].fwhm_x = in[i].fwhm_x; + out[i].fwhm_y = in[i].fwhm_y; + out[i].trate = in[i].trate; + out[i].ccdnum = in[i].ccdnum; + out[i].flags = in[i].flags; + out[i].imageID = in[i].imageID; + out[i].parentID = in[i].parentID; + out[i].externID = in[i].externID; + out[i].sourceID = in[i].sourceID; + + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].ubercalDist = in[i].ubercalDist; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; + } + return (out); +} + +Image_PS1_V3 *ImageInternalTo_PS1_V3 (Image *in, off_t Nvalues) { + + off_t i; + Image_PS1_V3 *out; + + ALLOCATE_ZERO (out, Image_PS1_V3, Nvalues); + + for (i = 0; i < Nvalues; i++) { + memcpy (&out[i].coords, &in[i].coords, sizeof(Coords)); + + strncpy (out[i].name, in[i].name, 120); // out[121], in[121] + out[i].name[120] = 0; // force termination + + out[i].tzero = in[i].tzero; + out[i].nstar = in[i].nstar; + out[i].secz = in[i].secz; + out[i].NX = in[i].NX; + out[i].NY = in[i].NY; + out[i].apmifit = in[i].apmifit; + out[i].dapmifit = in[i].dapmifit; + out[i].Mcal = in[i].Mcal; + out[i].dMcal = in[i].dMcal; + out[i].Xm = in[i].Xm; + out[i].photcode = in[i].photcode; + out[i].exptime = in[i].exptime; + out[i].sidtime = in[i].sidtime; + out[i].latitude = in[i].latitude; + + out[i].RAo = in[i].RAo; + out[i].DECo = in[i].DECo; + out[i].Radius = in[i].Radius; + + out[i].detection_limit = in[i].detection_limit; + out[i].saturation_limit = in[i].saturation_limit; + out[i].cerror = in[i].cerror; + out[i].fwhm_x = in[i].fwhm_x; + out[i].fwhm_y = in[i].fwhm_y; + out[i].trate = in[i].trate; + out[i].ccdnum = in[i].ccdnum; + out[i].flags = in[i].flags; + out[i].imageID = in[i].imageID; + out[i].parentID = in[i].parentID; + out[i].externID = in[i].externID; + out[i].sourceID = in[i].sourceID; + + // as of 2011.02.03, the old Mx,My,..., Mxxxx,Myyyy have been deprecated and replaced + // with the following. (no real databases used those values -- see + // libdvo/doc/dvo-images.txt) + out[i].nLinkAstrom = in[i].nLinkAstrom; + out[i].nLinkPhotom = in[i].nLinkPhotom; + out[i].ubercalDist = in[i].ubercalDist; + out[i].dXpixSys = in[i].dXpixSys; + out[i].dYpixSys = in[i].dYpixSys; + out[i].dMagSys = in[i].dMagSys; + out[i].nFitAstrom = in[i].nFitAstrom; + out[i].nFitPhotom = in[i].nFitPhotom; + out[i].photom_map_id = in[i].photom_map_id; + out[i].astrom_map_id = in[i].astrom_map_id; + } + return (out); +} + +PhotCode *PhotCode_PS1_V3_To_Internal (PhotCode_PS1_V3 *in, off_t Nvalues) { + + off_t i; + PhotCode *out; + + ALLOCATE_ZERO (out, PhotCode, Nvalues); + + for (i = 0; i < Nvalues; i++) { + strncpy (out[i].name, in[i].name, 31); // out[32], in[32] + out[i].name[31] = 0; // force termination + + out[i].code = in[i].code; + out[i].type = in[i].type; + out[i].C = in[i].C; + out[i].dC = in[i].dC; + out[i].dX = in[i].dX; + out[i].K = in[i].K; + out[i].c1 = in[i].c1; + out[i].c2 = in[i].c2; + out[i].equiv = in[i].equiv; + out[i].Nc = in[i].Nc; + memcpy (out[i].X, in[i].X, 4*sizeof(float)); + + out[i].astromErrSys = in[i].astromErrSys; + out[i].astromErrScale = in[i].astromErrScale; + out[i].astromErrMagScale = in[i].astromErrMagScale; + out[i].photomErrSys = in[i].photomErrSys; + + out[i].photomPoorMask = in[i].photomPoorMask; + out[i].photomBadMask = in[i].photomBadMask; + out[i].astromPoorMask = in[i].astromPoorMask; + out[i].astromBadMask = in[i].astromBadMask; + } + return (out); +} + +PhotCode_PS1_V3 *PhotCode_Internal_To_PS1_V3 (PhotCode *in, off_t Nvalues) { + + off_t i; + PhotCode_PS1_V3 *out; + + ALLOCATE_ZERO (out, PhotCode_PS1_V3, Nvalues); + + for (i = 0; i < Nvalues; i++) { + strncpy (out[i].name, in[i].name, 31); // out[32], in[32] + out[i].name[31] = 0; // force termination + + out[i].code = in[i].code; + out[i].type = in[i].type; + out[i].C = in[i].C; + out[i].dC = in[i].dC; + out[i].dX = in[i].dX; + out[i].K = in[i].K; + out[i].c1 = in[i].c1; + out[i].c2 = in[i].c2; + out[i].equiv = in[i].equiv; + out[i].Nc = in[i].Nc; + memcpy (out[i].X, in[i].X, 4*sizeof(float)); + + out[i].astromErrSys = in[i].astromErrSys; + out[i].astromErrScale = in[i].astromErrScale; + out[i].astromErrMagScale = in[i].astromErrMagScale; + out[i].photomErrSys = in[i].photomErrSys; + + out[i].photomPoorMask = in[i].photomPoorMask; + out[i].photomBadMask = in[i].photomBadMask; + out[i].astromPoorMask = in[i].astromPoorMask; + out[i].astromBadMask = in[i].astromBadMask; + } + return (out); +} Index: trunk/Ohana/src/libdvo/src/dvo_convert_elixir.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_elixir.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_elixir.c (revision 33649) @@ -158,4 +158,8 @@ out[i].Npos = 0.0; out[i].extID = 0; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -215,4 +219,8 @@ // changed or added for PS1_V2 out[i].flags = 0; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -271,5 +279,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -341,5 +349,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_loneos.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_loneos.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_loneos.c (revision 33649) @@ -150,4 +150,8 @@ out[i].Npos = 0.0; out[i].extID = 0; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -208,4 +212,8 @@ // changed or added for PS1_V2 out[i].flags = 0; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -264,5 +272,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -334,5 +342,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c (revision 33649) @@ -154,4 +154,8 @@ out[i].Npos = 0.0; out[i].extID = 0; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -215,4 +219,8 @@ // changed or added for PS1_V2 out[i].flags = 0; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -278,5 +286,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -347,5 +355,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c (revision 33649) @@ -154,4 +154,8 @@ out[i].Npos = 0.0; out[i].extID = 0; + + // added in PS1_V3 + out[i].photFlagsUpper = 0; + out[i].photFlagsLower = 0; } return (out); @@ -215,4 +219,8 @@ // changed or added for PS1_V2 out[i].flags = 0; + + // added in PS1_V3 + out[i].Map = 0; + out[i].ubercalDist = 0; } return (out); @@ -278,5 +286,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].ubercalDist = in[i].dummy3; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; @@ -347,5 +355,5 @@ out[i].nLinkAstrom = in[i].nLinkAstrom; out[i].nLinkPhotom = in[i].nLinkPhotom; - out[i].dummy3 = in[i].dummy3; + out[i].dummy3 = in[i].ubercalDist; // new name PS1_V3 out[i].dXpixSys = in[i].dXpixSys; out[i].dYpixSys = in[i].dYpixSys; Index: trunk/Ohana/src/libdvo/src/dvo_image.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_image.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_image.c (revision 33649) @@ -211,4 +211,5 @@ if (db[0].format == DVO_FORMAT_PS1_V1) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_V1"); if (db[0].format == DVO_FORMAT_PS1_V2) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_V2"); + if (db[0].format == DVO_FORMAT_PS1_V3) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_V3"); if (db[0].format == DVO_FORMAT_PS1_REF) gfits_modify (&db[0].header, "FORMAT", "%s", 1, "PS1_REF"); Index: trunk/Ohana/src/libdvo/src/dvo_image_raw.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_image_raw.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_image_raw.c (revision 33649) @@ -57,4 +57,5 @@ if (db[0].format == DVO_FORMAT_PS1_V1) ImageSize = sizeof(Image_PS1_V1); if (db[0].format == DVO_FORMAT_PS1_V2) ImageSize = sizeof(Image_PS1_V2); + if (db[0].format == DVO_FORMAT_PS1_V3) ImageSize = sizeof(Image_PS1_V3); if (db[0].format == DVO_FORMAT_PS1_REF) ImageSize = sizeof(Image_PS1_REF); @@ -85,4 +86,5 @@ if (db[0].format == DVO_FORMAT_PS1_V1) gfits_table_mkheader_Image_PS1_V1 (&db[0].theader); if (db[0].format == DVO_FORMAT_PS1_V2) gfits_table_mkheader_Image_PS1_V2 (&db[0].theader); + if (db[0].format == DVO_FORMAT_PS1_V3) gfits_table_mkheader_Image_PS1_V3 (&db[0].theader); if (db[0].format == DVO_FORMAT_PS1_REF) gfits_table_mkheader_Image_PS1_REF (&db[0].theader); Index: trunk/Ohana/src/libdvo/src/dvo_photcode_ops.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 33649) @@ -857,36 +857,4 @@ /***********************************************/ -// Create a map between the secfilt values from one photcode table to another -int *GetSecFiltMap(PhotCodeData *output, PhotCodeData *input) { - int i, j; - int NsecfiltIn = input[0].Nsecfilt; - int NsecfiltOut = output[0].Nsecfilt; - int *map; - ALLOCATE(map, int, NsecfiltIn); - - // loop over entries in the input table - for (i = 0; i < NsecfiltIn; i++) { - int code = input[0].codeNsec[i]; - int entry = -1; - // find the matching entry in the output table - for (j = 0; j < NsecfiltOut; j++) { - int outcode = output[0].codeNsec[j]; - if (code == outcode) { - entry = j; - break; - } - } - // if (entry == -1) { - // // entry is missing fail (no printfs in this file) - // free(map); - // return(FALSE); - // } - - // if entry is still -1, we will skip this one (not map into the output db) - map[i] = entry; - } - - return map; -} /* photcode table should have the following format: Index: trunk/Ohana/src/libdvo/src/dvo_photcode_utils.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_photcode_utils.c (revision 33649) +++ trunk/Ohana/src/libdvo/src/dvo_photcode_utils.c (revision 33649) @@ -0,0 +1,68 @@ +# include + +// Create a map between the secfilt values from one photcode table to another +int *GetSecFiltMap(PhotCodeData *output, PhotCodeData *input) { + int i, j; + int NsecfiltIn = input[0].Nsecfilt; + int NsecfiltOut = output[0].Nsecfilt; + int *map; + ALLOCATE(map, int, NsecfiltIn); + + // loop over entries in the input table + for (i = 0; i < NsecfiltIn; i++) { + int code = input[0].codeNsec[i]; + int entry = -1; + // find the matching entry in the output table + for (j = 0; j < NsecfiltOut; j++) { + int outcode = output[0].codeNsec[j]; + if (code == outcode) { + entry = j; + break; + } + } + // if (entry == -1) { + // // entry is missing fail (no printfs in this file) + // free(map); + // return(FALSE); + // } + + // if entry is still -1, we will skip this one (not map into the output db) + map[i] = entry; + } + + return map; +} + +PhotCode **ParsePhotcodeList (char *rawlist, int *nphotcodes, int needAve) { + + *nphotcodes = 0; + if (!rawlist) return NULL; + + int Nphotcodes = 0; + int NPHOTCODES = 10; + PhotCode **photcodes = NULL; + ALLOCATE (photcodes, PhotCode *, NPHOTCODES); + + /* parse the comma-separated list of photcodes */ + char *myList = strcreate(rawlist); + char *list = myList; + char *codename = NULL; + char *ptr = NULL; + while ((codename = strtok_r (list, ",", &ptr)) != NULL) { + list = NULL; // pass NULL on successive strtok_r calls + if ((photcodes[Nphotcodes] = GetPhotcodebyName (codename)) == NULL) { + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); + exit (1); + } + if (needAve && (photcodes[Nphotcodes][0].type != PHOT_SEC)) { + fprintf (stderr, "photcode %s is not an filter type (SEC)\n", codename); + exit (1); + } + Nphotcodes ++; + CHECK_REALLOCATE (photcodes, PhotCode *, NPHOTCODES, Nphotcodes, 10); + } + free (myList); + + *nphotcodes = Nphotcodes; + return photcodes; +} Index: trunk/Ohana/src/libdvo/src/dvo_tiny_values.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_tiny_values.c (revision 33649) +++ trunk/Ohana/src/libdvo/src/dvo_tiny_values.c (revision 33649) @@ -0,0 +1,75 @@ +# include "dvo.h" + +void CopyAverageToTiny (AverageTiny *averageT, Average *average) { + averageT[0].R = average[0].R; + averageT[0].D = average[0].D; + averageT[0].flags = average[0].flags; + averageT[0].Nmeasure = average[0].Nmeasure; + averageT[0].measureOffset = average[0].measureOffset; + averageT[0].catID = average[0].catID; + return; +} + +void CopyMeasureToTiny (MeasureTiny *measureT, Measure *measure) { + measureT[0].dR = measure[0].dR; + measureT[0].dD = measure[0].dD; + measureT[0].M = measure[0].M; + measureT[0].Mcal = measure[0].Mcal; + measureT[0].dM = measure[0].dM; + measureT[0].airmass = measure[0].airmass; + measureT[0].Xccd = measure[0].Xccd; + measureT[0].Yccd = measure[0].Yccd; + measureT[0].t = measure[0].t; + measureT[0].dt = measure[0].dt; + measureT[0].averef = measure[0].averef; + measureT[0].imageID = measure[0].imageID; + measureT[0].dbFlags = measure[0].dbFlags; + measureT[0].photFlags = measure[0].photFlags; + measureT[0].photcode = measure[0].photcode; + measureT[0].catID = measure[0].catID; + measureT[0].dXccd = measure[0].dXccd; + measureT[0].dYccd = measure[0].dYccd; + measureT[0].dRsys = measure[0].dRsys; + return ; +} + +// for the cases where we are not using a subset of the data, we still need to have a copy of these fields +int populate_tiny_values (Catalog *catalog, DVOTinyValueMode mode) { + + off_t i; + + AverageTiny *averageT; + Average *average; + + MeasureTiny *measureT; + Measure *measure; + + if (mode & DVO_TV_AVERAGE) { + ALLOCATE (catalog[0].averageT, AverageTiny, catalog[0].Naverage); + average = catalog[0].average; + averageT = catalog[0].averageT; + + for (i = 0; i < catalog[0].Naverage; i++) { + CopyAverageToTiny (&catalog[0].averageT[i], &catalog[0].average[i]); + } + } + + if (mode & DVO_TV_MEASURE) { + ALLOCATE (catalog[0].measureT, MeasureTiny, catalog[0].Nmeasure); + measure = catalog[0].measure; + measureT = catalog[0].measureT; + + for (i = 0; i < catalog[0].Nmeasure; i++) { + CopyMeasureToTiny (&catalog[0].measureT[i], &catalog[0].measure[i]); + } + } + return (TRUE); +} + +int free_tiny_values (Catalog *catalog) { + + if (catalog[0].averageT) free (catalog[0].averageT); + if (catalog[0].measureT) free (catalog[0].measureT); + return (TRUE); +} + Index: trunk/Ohana/src/libdvo/src/flatcorr_io.c =================================================================== --- trunk/Ohana/src/libdvo/src/flatcorr_io.c (revision 33649) +++ trunk/Ohana/src/libdvo/src/flatcorr_io.c (revision 33649) @@ -0,0 +1,201 @@ +# include "dvo.h" + +// FlatCorrection -- description of a single flat-field correction image +// a flat-field correction is defined for a specific camera (defined how?) or photcode (this makes some sense) +// it has a period of validity + +/* The flat-field correction is defined by two FITS tables / structures. The first + * (FlatCorrectionImage) describes the layout of the chips and which filters are being + * corrected. The second lists the actual corrections for each of the cells + */ + +FlatCorrectionTable *FlatCorrectionLoad (char *filename, int VERBOSE) { + + FILE *f; + Header header; + Matrix matrix; + Header theader; + FTable ftable; + FlatCorrectionTable *flatcorrTable; + + f = fopen (filename, "r"); + if (f == NULL) { + if (VERBOSE) fprintf (stderr, "can't find Flat Correction file %s\n", filename); + return (NULL); + } + + /* load in table data */ + ftable.header = &theader; + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read Flat Correction header\n"); + fclose (f); + return (NULL); + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read Flat Correction matrix\n"); + gfits_free_header (&header); + fclose (f); + return (NULL); + } + + ALLOCATE (flatcorrTable, FlatCorrectionTable, 1); + + if (!gfits_fread_ftable (f, &ftable, "FLAT_CORRECTION_IMAGE")) { + if (VERBOSE) fprintf (stderr, "can't read Flat Correction table\n"); + gfits_free_header (&header); + gfits_free_matrix (&matrix); + free (flatcorrTable); + fclose (f); + return (NULL); + } + flatcorrTable->image = gfits_table_get_FlatCorrectionImage (&ftable, &flatcorrTable->Nimage, NULL); + if (!flatcorrTable->image) { + fprintf (stderr, "ERROR: failed to read Flat Correction Images\n"); + exit (2); + } + gfits_free_header (&theader); + // NOTE: do not free ftable or the data will be freed + + if (!gfits_fread_ftable (f, &ftable, "FLAT_CORRECTION")) { + if (VERBOSE) fprintf (stderr, "can't read Flat Correction table\n"); + gfits_free_header (&header); + gfits_free_matrix (&matrix); + fclose (f); + return (NULL); + } + flatcorrTable->corr = gfits_table_get_FlatCorrection (&ftable, &flatcorrTable->Ncorr, NULL); + if (!flatcorrTable->corr) { + fprintf (stderr, "ERROR: failed to read Flat Corrections\n"); + exit (2); + } + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + // NOTE: do not free ftable or the data will be freed + + // convert the table format to map format: + FlatCorrectionInternal (flatcorrTable); + + return (flatcorrTable); +} + +int FlatCorrectionInternal(FlatCorrectionTable *flatcorrTable) { + + int i, j; + + // assert that the internal arrays are not yet allocated? + + // generate the arrays to hold the corrections & initialize + ALLOCATE (flatcorrTable->offset, float **, flatcorrTable->Nimage); + for (i = 0; i < flatcorrTable->Nimage; i++) { + ALLOCATE (flatcorrTable->offset[i], float *, flatcorrTable->image[i].Nx); + for (j = 0; j < flatcorrTable->image[i].Nx; j++) { + ALLOCATE (flatcorrTable->offset[i][j], float, flatcorrTable->image[i].Ny); + memset (flatcorrTable->offset[i][j], 0.0, flatcorrTable->image[i].Ny*sizeof(float)); + } + } + + // create the lookup table (ID -> Seq) + + // find the max value of ID + int MaxID = 0; + for (i = 0; i < flatcorrTable->Nimage; i++) { + MaxID = MAX(flatcorrTable->image[i].ID, MaxID); + } + MaxID ++; // we want the outer bound, not the last value + + // generate the index and init values to -1 + ALLOCATE (flatcorrTable->IDtoSeq, int, MaxID); + for (i = 0; i < MaxID; i++) { + flatcorrTable->IDtoSeq[i] = -1; + } + + // assign the ID values for each image + for (i = 0; i < flatcorrTable->Nimage; i++) { + int ID = flatcorrTable->image[i].ID; + assert (flatcorrTable->IDtoSeq[ID] == -1); + flatcorrTable->IDtoSeq[ID] = i; + } + + // assign the known specific correction values + for (i = 0; i < flatcorrTable->Ncorr; i++) { + int ID = flatcorrTable->corr[i].ID; + int x = flatcorrTable->corr[i].x; + int y = flatcorrTable->corr[i].y; + int seq = flatcorrTable->IDtoSeq[ID]; + assert (seq != -1); + + flatcorrTable->offset[seq][x][y] = flatcorrTable->corr[i].offset; + } + + return TRUE; +} + +int FlatCorrectionSave (FlatCorrectionTable *flatcorrTable, char *filename) { + + Header header; + Matrix matrix; + Header theaderImage; + FTable ftableImage; + + Header theaderCorr; + FTable ftableCorr; + FILE *f; + + FlatCorrectionImage *image; + FlatCorrection *corr; + + // these output functions byteswap their buffers. make a copy for output + ALLOCATE (image, FlatCorrectionImage, flatcorrTable->Nimage); + memcpy (image, flatcorrTable->image, flatcorrTable->Nimage*sizeof(FlatCorrectionImage)); + + ALLOCATE (corr, FlatCorrection, flatcorrTable->Ncorr); + memcpy (corr, flatcorrTable->corr, flatcorrTable->Ncorr*sizeof(FlatCorrection)); + + /* make phu header (no matrix needed) */ + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + ftableImage.header = &theaderImage; + gfits_table_set_FlatCorrectionImage (&ftableImage, image, flatcorrTable->Nimage); + + ftableCorr.header = &theaderCorr; + gfits_table_set_FlatCorrection (&ftableCorr, corr, flatcorrTable->Ncorr); + + f = fopen (filename, "w"); + if (f == (FILE *) NULL) { + fprintf (stderr, "cannot open %s for output\n", filename); + return (FALSE); + } + + gfits_fwrite_header (f, &header); + gfits_fwrite_matrix (f, &matrix); + gfits_fwrite_Theader (f, &theaderImage); + gfits_fwrite_table (f, &ftableImage); + gfits_fwrite_Theader (f, &theaderCorr); + gfits_fwrite_table (f, &ftableCorr); + fclose (f); + + return (TRUE); +} + +float FlatCorrectionOffset (FlatCorrectionTable *flatcorr, int ID, int X, int Y) { + + // validate the flat_id (not out of range?) + int seq = flatcorr->IDtoSeq[ID]; + + // convert X,Y to Xbin, Ybin: + int Xbin = MAX(MIN(X * flatcorr->image[seq].Nx / flatcorr->image[seq].DX, flatcorr->image[seq].Nx - 1), 0); + int Ybin = MAX(MIN(Y * flatcorr->image[seq].Ny / flatcorr->image[seq].DY, flatcorr->image[seq].Ny - 1), 0); + + // XXX warn if X,Y are out of range? not super important unless *way* out of range + + float offset = flatcorr->offset[seq][Xbin][Ybin]; + return offset; +} + +// other needed operations: +// * assign an image to a flatcorr (based on photcode and obs_date) Index: trunk/Ohana/src/libdvo/src/skyregion_gsc.c =================================================================== --- trunk/Ohana/src/libdvo/src/skyregion_gsc.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/skyregion_gsc.c (revision 33649) @@ -91,4 +91,7 @@ L0.regions[0].childS = 0; L0.regions[0].childE = NDECBANDS; + L0.regions[0].hostFlags = 0; + L0.regions[0].hostID = 0; + L0.regions[0].backupID = 0; strcpy (L0.regions[0].name, "fullsky"); if (DEBUG) SkyRegionPrint (&L0.regions[0]); @@ -117,4 +120,7 @@ L1.regions[i].child = TRUE; L1.regions[i].table = -1; + L1.regions[i].hostFlags = 0; + L1.regions[i].hostID = 0; + L1.regions[i].backupID = 0; strcpy (L1.regions[i].name, DecNames[i]); if (DEBUG) SkyRegionPrint (&L1.regions[i]); @@ -152,4 +158,7 @@ ALLOCATE (skytable[0].regions, SkyRegion, 1); skytable[0].Nregions = 0; + + // define the host table filename + strcpy (skytable->hosts, "HostTable.dat"); // L0, L1, L2, L3, L4 all have index ranges of 0 -> Nregions and parent values pointing @@ -417,4 +426,7 @@ L2[0].regions[Nr].childS = 0; L2[0].regions[Nr].childE = 0; + L2[0].regions[Nr].hostFlags = 0; + L2[0].regions[Nr].hostID = 0; + L2[0].regions[Nr].backupID = 0; /* define names for L2 regions */ @@ -464,4 +476,7 @@ L3[0].regions[Nr].childS = 0; L3[0].regions[Nr].childE = 0; + L3[0].regions[Nr].hostFlags = 0; + L3[0].regions[Nr].hostID = 0; + L3[0].regions[Nr].backupID = 0; if (DEBUG >= 3) SkyRegionPrint (&L3[0].regions[Nr]); @@ -514,4 +529,7 @@ L4[0].regions[Nr].childS = 0; L4[0].regions[Nr].childE = 0; + L4[0].regions[Nr].hostFlags = 0; + L4[0].regions[Nr].hostID = 0; + L4[0].regions[Nr].backupID = 0; sprintf (name, "%s.%02d", L3[0].name, Nbox); Index: trunk/Ohana/src/libdvo/src/skyregion_io.c =================================================================== --- trunk/Ohana/src/libdvo/src/skyregion_io.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/skyregion_io.c (revision 33649) @@ -39,4 +39,5 @@ ALLOCATE (skytable, SkyTable, 1); + memset (skytable->hosts, 0, 80); skytable[0].regions = gfits_table_get_SkyRegion (&ftable, &skytable[0].Nregions, NULL); if (!skytable[0].regions) { @@ -50,4 +51,28 @@ } + // 2012.02.08 : I updated the schema for SkyTable.dat to add 3 fields by stealing the + // last 3 bytes of the 'name' field (going from 21 bytes to 18 bytes). This was safe + // because no files had yet been defined with more than 16 bytes in the names, leave + // room for 1 more byte includig the EOF char. + + // When we load old tables, we need to zero out the values of those 3 new fields (they + // may have garbage left over). To distinguish old and new tables, I add the field + // 'HOSTS' to the PHU header (which also tells the location of the host table, if used). + + int haveHosts = gfits_scan (&header, "HOSTS", "%s", 1, skytable->hosts); + if (!haveHosts) { + strcpy (skytable->hosts, "HostTable.dat"); + for (i = 0; i < skytable[0].Nregions; i++) { + if (strlen(skytable[0].regions[i].name) > 17) { + fprintf (stderr, "WARNING : this skytable has some names longer than the adjusted size of 17 bytes.\n"); + fprintf (stderr, " This is not compatible with the current SkyTable definition\n"); + exit (1); + } + skytable[0].regions[i].hostFlags = 0; + skytable[0].regions[i].hostID = 0; + skytable[0].regions[i].backupID = 0; + } + } + gfits_free_header (&header); gfits_free_matrix (&matrix); @@ -71,4 +96,5 @@ gfits_create_header (&header); gfits_create_matrix (&header, &matrix); + gfits_print (&header, "HOSTS", "%s", 1, skytable->hosts); gfits_table_set_SkyRegion (&ftable, skytable[0].regions, skytable[0].Nregions); @@ -91,10 +117,10 @@ SkyTable *SkyTableLoadOptimal (char *catdir, char *skyfile, char *gscfile, int readwrite, int depth, int verbose) { - char filename[256]; + char *filename; struct stat filestat; SkyTable *sky; - /* first option: CATDIR/SkyTable.fits */ - sprintf (filename, "%s/SkyTable.fits", catdir); + /* first option: in CATDIR */ + filename = SkyTableFilename (catdir); if (stat (filename, &filestat)) goto SKYFILE; if (!check_file_access (filename, FALSE, readwrite, verbose)) goto SKYFILE; @@ -115,4 +141,5 @@ if (sky == NULL) { fprintf (stderr, "error loading sky table\n"); + free (filename); return (NULL); } @@ -121,9 +148,12 @@ /* create CATDIR copy */ - sprintf (filename, "%s/SkyTable.fits", catdir); check_file_access (filename, FALSE, readwrite, verbose); - if (!SkyTableSave (sky, filename)) return NULL; + if (!SkyTableSave (sky, filename)) { + free (filename); + return NULL; + } gfits_convert_SkyRegion (sky[0].regions, sizeof (SkyTable), sky[0].Nregions); + free (filename); return (sky); @@ -134,4 +164,5 @@ if (gscfile == NULL) { fprintf (stderr, "error loading sky table from existing db (%s)\n", catdir); + free (filename); return (NULL); } @@ -140,13 +171,17 @@ if (sky == NULL) { fprintf (stderr, "error loading sky table\n"); + free (filename); return (NULL); } /* create CATDIR copy */ - sprintf (filename, "%s/SkyTable.fits", catdir); check_file_access (filename, FALSE, readwrite, verbose); - if (!SkyTableSave (sky, filename)) return NULL; + if (!SkyTableSave (sky, filename)) { + free (filename); + return NULL; + } gfits_convert_SkyRegion (sky[0].regions, sizeof (SkyRegion), sky[0].Nregions); + free (filename); return (sky); } @@ -180,2 +215,13 @@ } +// given a CATDIR, return an allocated string with the full skytable name +char *SkyTableFilename (char *catdir) { + + if (!catdir) return NULL; + + char *skyfile; + ALLOCATE (skyfile, char, strlen(catdir) + strlen("/SkyTable.fits") + 1); + sprintf (skyfile, "%s/SkyTable.fits", catdir); + + return skyfile; +} Index: trunk/Ohana/src/libdvo/src/skyregion_ops.c =================================================================== --- trunk/Ohana/src/libdvo/src/skyregion_ops.c (revision 33648) +++ trunk/Ohana/src/libdvo/src/skyregion_ops.c (revision 33649) @@ -20,4 +20,5 @@ list[0].Nregions = 0; list[0].ownElements = FALSE; // this list is only holding a view to the elements + strcpy (list[0].hosts, table[0].hosts); region = table[0].regions; @@ -67,4 +68,5 @@ list[0].Nregions = N; list[0].ownElements = FALSE; // this list is only holding a view to the elements + strcpy (list[0].hosts, table[0].hosts); region = table[0].regions; @@ -102,4 +104,5 @@ list[0].Nregions = N; list[0].ownElements = FALSE; // this list is only holding a view to the elements + strcpy (list[0].hosts, table[0].hosts); region = table[0].regions; @@ -261,4 +264,5 @@ ALLOCATE (list[0].filename, char *, NNEW); list[0].ownElements = FALSE; // this list is only holding a view to the elements + strcpy (list[0].hosts, table[0].hosts); region = table[0].regions; @@ -326,4 +330,5 @@ list[0].Nregions = 0; list[0].ownElements = FALSE; // this list is only holding a view to the elements + strcpy (list[0].hosts, inList[0].hosts); region = inList[0].regions; @@ -403,4 +408,5 @@ ALLOCATE (list[0].filename, char *, NNEW); list[0].ownElements = FALSE; // this list is only holding a view to the elements + strcpy (list[0].hosts, table[0].hosts); region = table[0].regions;