Index: trunk/ippToPsps/src/DetectionBatch.c =================================================================== --- trunk/ippToPsps/src/DetectionBatch.c (revision 30192) +++ trunk/ippToPsps/src/DetectionBatch.c (revision 30193) @@ -1,4 +1,8 @@ #include "DetectionBatch.h" #include "DetectionBatchEnums.h" + +/** + Source for all methods in the DetectionBatch class + */ /** @@ -14,5 +18,5 @@ /** - Does the work. + Does the work. Loops through the provided smf files, tallies the contents with the DVO database already connected to and generates a new FITS file of the fomat required by PSPS (i.e. a binary table per PSPS database table) */ static int run(DetectionBatch* this) { @@ -82,5 +86,5 @@ // stuff for detections table - uint32_t s,d, invalidDvoRows, unmatched, totalDetections = 0; + uint32_t s,d, invalidDvoRows, nChipDetectionsOut = 0; long longnull = -999; @@ -122,7 +126,6 @@ short nOta = 0; long i; - bool isDuplicate; uint32_t numOfDuplicates, numInvalidFlux, numOfInvalidIppIDet; - long numDetectionsOut, totalDetectionsOut = 0, numPhotoRef, totalNumPhotoRef = 0; + long totalDetectionsOut = 0, numPhotoRef, totalNumPhotoRef = 0; long* removeList = (long*)calloc(this->MAXDETECT, sizeof(long)); long thisObjId; @@ -220,11 +223,11 @@ nOta++; - long nDet = 0; - if (fits_get_num_rows(fitsIn, &nDet, &status)) { + long nChipDetectionsIn = 0; + if (fits_get_num_rows(fitsIn, &nChipDetectionsIn, &status)) { fits_report_error(stderr, status); } // loop round detections to populate some extra stuff - s = d = totalDetections = invalidDvoRows = unmatched = 0; + s = d = nChipDetectionsOut = invalidDvoRows = 0; // determine column numbers for certain IPP detection columns @@ -244,8 +247,8 @@ anynull = 0; - fits_read_col(fitsIn, TLONG, ippIDetNum, 1, 1, nDet, &longnull, ippIDet, &anynull, &status); - fits_read_col(fitsIn, TFLOAT, instMagNum, 1, 1, nDet, &floatnull, instMag, &anynull, &status); - fits_read_col(fitsIn, TFLOAT, instMagErrNum, 1, 1, nDet, &floatnull, instMagErr, &anynull, &status); - fits_read_col(fitsIn, TFLOAT, peakMagNum, 1, 1, nDet, &floatnull, peakMag, &anynull, &status); + fits_read_col(fitsIn, TLONG, ippIDetNum, 1, 1, nChipDetectionsIn, &longnull, ippIDet, &anynull, &status); + fits_read_col(fitsIn, TFLOAT, instMagNum, 1, 1, nChipDetectionsIn, &floatnull, instMag, &anynull, &status); + fits_read_col(fitsIn, TFLOAT, instMagErrNum, 1, 1, nChipDetectionsIn, &floatnull, instMagErr, &anynull, &status); + fits_read_col(fitsIn, TFLOAT, peakMagNum, 1, 1, nChipDetectionsIn, &floatnull, peakMag, &anynull, &status); // DVO detections are ordered by IPP_IDET, which increments from 0 in SMF table @@ -254,26 +257,34 @@ numInvalidFlux = 0; numOfInvalidIppIDet = 0; - for (long s = 0; s max DVO det ID) + // 3. it has an invalid flux value + for (long s = 0; s maxDvoDetId) { removeList[numOfDuplicates+numInvalidFlux+numOfInvalidIppIDet] = s+1; numOfInvalidIppIDet++; + continue; } - else { - while (ippIDet[s] > dvoDetections[d].meas.detID && dvoDetections[d].meas.detID <= maxDvoDetId) { - - d++; - if (!dvoDetections[d].valid) invalidDvoRows++; - } + + // loop forward through DVO detections until we find corresponding detection ID + while (ippIDet[s] > dvoDetections[d].meas.detID && dvoDetections[d].meas.detID <= maxDvoDetId) { + + d++; + if (!dvoDetections[d].valid) invalidDvoRows++; } + + // if we have found matching detection IDs then continue to populate other columns if (ippIDet[s] == dvoDetections[d].meas.detID ) { - // TODO HACK duplicates bug - don't include any duplicates + // check if we have already seen this obj ID, and if we have, skip this detection thisObjId = dvoDetections[d].ave.extID; - for (i=0; i maxObjID) maxObjID = objID[s]; - if (objID[s] < minObjID) minObjID = objID[s]; - } - totalDetections++; + + // store max/min objID + if (objID[s] > maxObjID) maxObjID = objID[s]; + if (objID[s] < minObjID) minObjID = objID[s]; + + nChipDetectionsOut++; } - // if we get here then we cannot find this smf detection in DVO - else { - - unmatched++; - continue; - } - - if (dvoDetections[d].meas.detID == maxDvoDetId) break; - } + } // end of loop round detections // write number of rows (detections) to ImageMeta - numDetectionsOut = totalDetections - numOfDuplicates - numInvalidFlux; - fits_write_col(this->base.fitsOut, TLONG, IMAGEMETA_NDETECT, 1, 1, 1, &numDetectionsOut, &status); + fits_write_col(this->base.fitsOut, TLONG, IMAGEMETA_NDETECT, 1, 1, 1, &nChipDetectionsOut, &status); + + int totalRemove = numOfDuplicates+numInvalidFlux+numOfInvalidIppIDet; // avoid writing out an empty table of detections - if (numDetectionsOut > 0) { + if (nChipDetectionsOut > 0) { // detections - ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nDet, "Detection", false); - fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_OBJID, 1, 1, nDet, objID, &status); - fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_DETECTID, 1, 1, nDet, detectID, &status); - fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IPPOBJID, 1, 1, nDet, ippObjID, &status); - fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IPPDETECTID, 1, 1, nDet, ippDetectID, &status); - fits_write_col(this->base.fitsOut, TBYTE, DETECTION_FILTERID, 1, 1, nDet, filterIDs, &status); - fits_write_col(this->base.fitsOut, TBYTE, DETECTION_SURVEYID, 1, 1, nDet, surveyIDs, &status); - fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IMAGEID, 1, 1, nDet, imageID, &status); - fits_write_col(this->base.fitsOut, TDOUBLE, DETECTION_OBSTIME, 1, 1, nDet, obsTimes, &status); - fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_INSTFLUX, 1, 1, nDet, instFlux, &status); - fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_INSTFLUXERR, 1, 1, nDet, instFluxErr, &status); - fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_PEAKADU, 1, 1, nDet, peakFlux, &status); - fits_write_col(this->base.fitsOut, TSTRING, DETECTION_ASSOCDATE, 1, 1, nDet, assocDate, &status); - fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_INFOFLAG, 1, 1, nDet, flags, &status); - if (numOfDuplicates||numInvalidFlux) fits_delete_rowlist(this->base.fitsOut, removeList, numOfDuplicates+numInvalidFlux, &status); + ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nChipDetectionsIn, "Detection", false); + fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_OBJID, 1, 1, nChipDetectionsIn, objID, &status); + fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_DETECTID, 1, 1, nChipDetectionsIn, detectID, &status); + fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IPPOBJID, 1, 1, nChipDetectionsIn, ippObjID, &status); + fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IPPDETECTID, 1, 1, nChipDetectionsIn, ippDetectID, &status); + fits_write_col(this->base.fitsOut, TBYTE, DETECTION_FILTERID, 1, 1, nChipDetectionsIn, filterIDs, &status); + fits_write_col(this->base.fitsOut, TBYTE, DETECTION_SURVEYID, 1, 1, nChipDetectionsIn, surveyIDs, &status); + fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IMAGEID, 1, 1, nChipDetectionsIn, imageID, &status); + fits_write_col(this->base.fitsOut, TDOUBLE, DETECTION_OBSTIME, 1, 1, nChipDetectionsIn, obsTimes, &status); + fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_INSTFLUX, 1, 1, nChipDetectionsIn, instFlux, &status); + fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_INSTFLUXERR, 1, 1, nChipDetectionsIn, instFluxErr, &status); + fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_PEAKADU, 1, 1, nChipDetectionsIn, peakFlux, &status); + fits_write_col(this->base.fitsOut, TSTRING, DETECTION_ASSOCDATE, 1, 1, nChipDetectionsIn, assocDate, &status); + fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_INFOFLAG, 1, 1, nChipDetectionsIn, flags, &status); + if (numOfDuplicates||numInvalidFlux||numOfInvalidIppIDet) + fits_delete_rowlist(this->base.fitsOut, removeList, totalRemove, &status); // skinny object - ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nDet, "SkinnyObject", false); - fits_write_col(this->base.fitsOut, TLONGLONG, SKINNYOBJECT_OBJID, 1, 1, nDet, objID, &status); - fits_write_col(this->base.fitsOut, TLONGLONG, SKINNYOBJECT_IPPOBJID, 1, 1, nDet, ippObjID, &status); - fits_write_col(this->base.fitsOut, TBYTE, SKINNYOBJECT_SURVEYID, 1, 1, nDet, surveyIDs, &status); - if (numOfDuplicates||numInvalidFlux) fits_delete_rowlist(this->base.fitsOut, removeList, numOfDuplicates+numInvalidFlux, &status); + ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nChipDetectionsIn, "SkinnyObject", false); + fits_write_col(this->base.fitsOut, TLONGLONG, SKINNYOBJECT_OBJID, 1, 1, nChipDetectionsIn, objID, &status); + fits_write_col(this->base.fitsOut, TLONGLONG, SKINNYOBJECT_IPPOBJID, 1, 1, nChipDetectionsIn, ippObjID, &status); + fits_write_col(this->base.fitsOut, TBYTE, SKINNYOBJECT_SURVEYID, 1, 1, nChipDetectionsIn, surveyIDs, &status); + if (numOfDuplicates||numInvalidFlux||numOfInvalidIppIDet) + fits_delete_rowlist(this->base.fitsOut, removeList, totalRemove, &status); // object calibration color - ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nDet, "ObjectCalColor", false); - fits_write_col(this->base.fitsOut, TLONGLONG, OBJECTCALCOLOR_OBJID, 1, 1, nDet, objID, &status); - fits_write_col(this->base.fitsOut, TLONGLONG, OBJECTCALCOLOR_IPPOBJID, 1, 1, nDet, ippObjID, &status); - fits_write_col(this->base.fitsOut, TBYTE, OBJECTCALCOLOR_FILTERID, 1, 1, nDet, filterIDs, &status); - if (numOfDuplicates||numInvalidFlux) fits_delete_rowlist(this->base.fitsOut, removeList, numOfDuplicates+numInvalidFlux, &status); + ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nChipDetectionsIn, "ObjectCalColor", false); + fits_write_col(this->base.fitsOut, TLONGLONG, OBJECTCALCOLOR_OBJID, 1, 1, nChipDetectionsIn, objID, &status); + fits_write_col(this->base.fitsOut, TLONGLONG, OBJECTCALCOLOR_IPPOBJID, 1, 1, nChipDetectionsIn, ippObjID, &status); + fits_write_col(this->base.fitsOut, TBYTE, OBJECTCALCOLOR_FILTERID, 1, 1, nChipDetectionsIn, filterIDs, &status); + if (numOfDuplicates||numInvalidFlux||numOfInvalidIppIDet) + fits_delete_rowlist(this->base.fitsOut, removeList, totalRemove, &status); } psLogMsg("ippToPsps", PS_LOG_INFO, - "+-----------+---------+----------+------------------+---------------+--------------+\n" - "| Extension | Rows in | Rows out | Missing from DVO | Duplicate IDs | Invalid Flux |\n" - "| %s | %5ld | %5ld | %5d | %5d | %5d |\n", - extensionName, nDet, numDetectionsOut, unmatched, numOfDuplicates, numInvalidFlux); - - totalDetectionsOut += numDetectionsOut; + "+-----------+---------+----------+----------------+--------------+--------------+\n" + "| Extension | Rows in | Rows out | Duplicate IDs | Invalid Flux | Bogus det ID |\n" + "| %s | %5ld | %5d | %5d | %5d | %5d |\n", + extensionName, nChipDetectionsIn, nChipDetectionsOut, numOfDuplicates, numInvalidFlux, numOfInvalidIppIDet); + + totalDetectionsOut += nChipDetectionsOut; if (dvoDetections!= NULL) dvoFree(dvoDetections); @@ -492,5 +501,5 @@ /** - Constructor + Constructor. Returns a new DetectionBatch object. */ DetectionBatch* new_DetectionBatch(int *argc, char **argv) {