Index: /trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- /trunk/ippScripts/scripts/addstar_run.pl (revision 37550) +++ /trunk/ippScripts/scripts/addstar_run.pl (revision 37551) @@ -94,7 +94,11 @@ # Recipes to use based on reduction class $reduction = 'DEFAULT' unless defined $reduction; -if ($stage =~/skycal/) { - $reduction = 'DEFAULT'; -} +if ($stage =~/diff/) { + $reduction = 'ADDSTAR'; +} +if ($stage =~/forcedwarp/) { + $reduction = 'ADDSTAR'; +} + my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # XXX This is now not used: do we still need it? @@ -132,4 +136,11 @@ print "using $fpaObjects for $stage\n"; } +if ($stage =~/diff/) { + print "using $fpaObjects for $stage\n"; +} +if ($stage =~/forcedwarp/) { + print "using $fpaObjects for $stage\n"; +} + @@ -177,5 +188,5 @@ } -if (($stage =~/stack/ || $stage =~ /skycal/)) { +if (($stage =~/stack/ || $stage =~ /skycal/ || $stage =~/diff/ || $stage=~/forcedwarp/)) { $fpaObjects =~ s/smf$/cmf/; $fpaObjectsAlt =~ s/smf$/cmf/; @@ -305,4 +316,10 @@ $command .= " -quick-airmass"; } #careful here - this matches staticsky and staticsky_multi + if ($stage =~ /diff/) { + $command .= " -accept-astrom"; + } + if ($stage =~ /forcedwarp/) { + $command .= " -accept-astrom -xrad"; + } my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script Index: /trunk/ippTasks/addstar.pro =================================================================== --- /trunk/ippTasks/addstar.pro (revision 37550) +++ /trunk/ippTasks/addstar.pro (revision 37551) @@ -52,4 +52,10 @@ active false end + task addstar.revert.diff + active false + end + task addstar.revert.fullforce + active false + end end @@ -65,4 +71,10 @@ end task addstar.revert.skycal + active true + end + task addstar.revert.diff + active true + end + task addstar.revert.fullforce active true end @@ -141,4 +153,6 @@ $addstar_revert_DB_SSM = 0 $addstar_revert_DB_SC = 0 +$addstar_revert_DB_FF = 0 +$addstar_revert_DB_DF = 0 #$LOADEXEC_ADD = 300 @@ -275,4 +289,10 @@ sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID end + if ("$STAGE" == "diff") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "fullforce") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end @@ -293,4 +313,10 @@ if ("$STAGE" == "skycal") $run = $run --stage_id $STAGE_ID + end + if ("$STAGE" == "diff") + $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 + end + if ("$STAGE" == "fullforce") + $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 end @@ -526,2 +552,98 @@ end end + + +task addstar.revert.diff + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage diff + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_DF + $run = $run -dbname $DB:$addstar_revert_DB_DF + $addstar_revert_DB_DF ++ + if ($addstar_revert_DB_DF >= $DB:n) set addstar_revert_DB_DF = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +task addstar.revert.fullforce + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage fullforce + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_FF + $run = $run -dbname $DB:$addstar_revert_DB_FF + $addstar_revert_DB_FF ++ + if ($addstar_revert_DB_FF >= $DB:n) set addstar_revert_DB_FF = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /trunk/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- /trunk/ippToPsps/config/ippToPspsDbSchema.sql (revision 37550) +++ /trunk/ippToPsps/config/ippToPspsDbSchema.sql (revision 37551) @@ -189,4 +189,19 @@ SET character_set_client = @saved_cs_client; -- +-- Table structure for table `forcedobject` +-- + +DROP TABLE IF EXISTS `forcedobject`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `forcedobject` ( + `batch_id` bigint(20) unsigned NOT NULL, + `region` varchar(50) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `diffobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- -- Table structure for table `diffobject` -- Index: /trunk/ippToPsps/jython/Makefile.am =================================================================== --- /trunk/ippToPsps/jython/Makefile.am (revision 37550) +++ /trunk/ippToPsps/jython/Makefile.am (revision 37551) @@ -14,4 +14,6 @@ diffobjectbatch.py \ dvodetections.py \ + dvoforceddetections.py \ + dvoforcedobjects.py \ dvoobjects.py \ dvo.py \ Index: /trunk/ippToPsps/jython/batch.py =================================================================== --- /trunk/ippToPsps/jython/batch.py (revision 37550) +++ /trunk/ippToPsps/jython/batch.py (revision 37551) @@ -579,5 +579,5 @@ # for OB, this imports object data from DVO if not self.importIppTables(): - if self.batchType == "OB": + if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO"): self.logger.errorPair("skipping this batch", "no dvo tables for this region") self.ippToPspsDb.updateProcessed(self.batchID, 1) Index: /trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- /trunk/ippToPsps/jython/detectionbatch.py (revision 37550) +++ /trunk/ippToPsps/jython/detectionbatch.py (revision 37551) @@ -214,5 +214,5 @@ sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT')) sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR')) - sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) + sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')) sqlLine.group("numAstroRef", self.getKeyValue(header, 'NASTRO')) sqlLine.group("numPhotoRef", self.getKeyValue(header, 'NASTRO')) Index: /trunk/ippToPsps/jython/dvoforceddetections.py =================================================================== --- /trunk/ippToPsps/jython/dvoforceddetections.py (revision 37551) +++ /trunk/ippToPsps/jython/dvoforceddetections.py (revision 37551) @@ -0,0 +1,178 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob + +from dvo import Dvo + + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoForcedDetections(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): + + super(DvoForcedDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + # declare DVO file types of interest + self.ingestFileTypes = ['cpm','cpt','cpx'] + + ''' + Ingests detections for this region + ''' + def ingestRegion(self, region): + + cpmPath = self.skychunk.dvoLocation + "/" + region + ".cpm" + cptPath = self.skychunk.dvoLocation + "/" + region + ".cpt" + cpxPath = self.skychunk.dvoLocation + "/" + region + ".cpx" + + cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm") + cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt") + cpxTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpx") + + # if we have not already ingested the cpm table, then do it now + if not self.scratchDb.alreadyImportedThisDvoTable(cpxPath): + + # import cpm table and index + self.importFits( + cpmPath, + "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID AIRMASS M_CAL D_RA D_DEC PHOTCODE", + cpmTableName) + + self.scratchDb.createIndex(cpmTableName, "IMAGE_ID") + self.scratchDb.createIndex(cpmTableName, "CAT_ID") + self.scratchDb.createIndex(cpmTableName, "OBJ_ID") + + # XXX: Should we create index on PHOTCODE? + + # we can now report that we have imported this table + self.scratchDb.setImportedThisDvoTable(cpmPath, region) + + # get the id for this cpm file so we can insert it in the dvoDetection table + fileId = self.scratchDb.getDvoFileId(cpmPath) + + # import cpt table and index + self.importFits( + cptPath, + "OBJ_ID CAT_ID EXT_ID RA DEC" + cptTableName) + + self.scratchDb.createIndex(cptTableName, "IMAGE_ID") + self.scratchDb.createIndex(cptTableName, "CAT_ID") + self.scratchDb.createIndex(cptTableName, "OBJ_ID") + + # heather thinks this may be a problem (but I'm not entirely sure) - why create index on image_id? it's not an imported column. + + self.importFits( + cpxPath, + "X11_SM_OBJ X12_SM_OBJ X22_SM_OBJ E1_SM_OBJ E2_SM_OBJ X11_SH_OBJ X12_SH_OBJ X22_SH_OBJ E1_SH_OBJ E2_SH_OBJ \ + X11_SM_PSF X12_SM_PSF X22_SM_PSF E1_SM_PSF E2_SM_PSF X11_SH_PSF X12_SH_PSF X22_SH_PSF E1_SH_PSF E2_SH_PSF \ + FLUX_AP_R5 FLUX_ERR_AP_R5 FLUX_STD_AP_R5 FLUX_FIL_AP_R5 FLUX_AP_R6 FLUX_ERR_AP_R6 FLUX_STD_AP_R6 FLUX_FIL_AP_R6 \ + DET_ID OBJ_ID CAT_ID IMAGE_ID", + cpxTableName) + self.scratchDb.createIndex(cpxTableName, "IMAGE_ID") + self.scratchDb.createIndex(cpxTableName, "CAT_ID") + self.scratchDb.createIndex(cpxTableName, "OBJ_ID") + + + + + self.logger.infoPair("Adding","PSPS objIDs") + # first try to add the new columns. catch failure and continue + try: + sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (PSPS_OBJ_ID BIGINT), ADD COLUMN (ZP REAL), ADD COLUMN (RA FLOAT), ADD COLUMN (DEC_ FLOAT)" + self.scratchDb.execute(sql) + except: + # failed, but may be due to mysql error + self.logger.infoPair("mysql error or already added PSPS_OBJ_ID column to", cpmTableName) + # self.logger.infoPair("sql command", sql) + + + # shove PSPS objIDs from cpt table and the calibrated zero point and coordinates which are formed + # by combining values from multiple tables into cpt table + sql = "UPDATE " \ + + cpmTableName + " AS meas, " \ + + cptTableName + " AS ave, " \ + + self.scratchDb.dvoPhotcodesTable + " AS phot \ + SET \ + meas.PSPS_OBJ_ID = ave.EXT_ID, \ + meas.ZP = phot.C_LAM * 0.001 + phot.K * (meas.AIRMASS - 1) - meas.M_CAL, \ + meas.RA = ave.RA - (meas.D_RA / 3600.0), \ + meas.DEC_ = ave.DEC_ - (meas.D_DEC / 3600.0) \ + WHERE meas.CAT_ID = ave.CAT_ID \ + AND meas.OBJ_ID = ave.OBJ_ID \ + AND meas.PHOTCODE = phot.CODE" + + # EAM : add some debug I/O + # self.logger.infoPair("sql command", sql) + self.scratchDb.execute(sql) + +# HAF -- need to find out ZP Nominal and to do the columns here, otherwise this is done (up above is done) + + + # NB we use an INSERT IGNORE here. This is because of a known issue where multiple DVO cpm + # files can include the same detection, with the same IMAGE_ID/DET_ID pairing, but different + # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary + # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest + # XXX: Bills 2012-10031 Does this problem still exist? + self.logger.infoPair("Merging into", self.scratchDb.dvoDetectionTable) + sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\ + imageID \ + ,ippDetectID \ + ,detectID \ + ,ippObjID \ + ,objID \ + ,flags \ + ,zp \ + ,zpErr \ + ,airMass \ + ,expTime \ + ,ra \ + ,dec_ \ + ,raErr \ + ,decErr \ + ,fileID \ + ) SELECT \ + IMAGE_ID \ + ,DET_ID \ + ,EXT_ID \ + ,CAT_ID * 1000000000 + OBJ_ID \ + ,PSPS_OBJ_ID \ + ,DB_FLAGS \ + ,ZP \ + ,MAG_CAL_ERR \ + ,AIRMASS \ + ,POW(10.0, 0.4 * M_TIME) \ + ,RA \ + ,DEC_ \ + ,X_CCD_ERR*PLTSCALE \ + ,Y_CCD_ERR*PLTSCALE \ + , " + str(fileId) + " \ + FROM " + cpmTableName + + # print "sql :", sql + # response = raw_input("prepare to insert dvoDetectionTable ") + + try: + self.scratchDb.execute(sql) + except: + self.logger.error("FAILED: " + sql) + return False + finally: + # response = raw_input("insert dvoDetectionTable ") + self.logger.infoPair("Dropping tables", "%s, %s and %s " % (cpmTableName, cptTableName, cpxTableName)) + self.scratchDb.dropTable(cpmTableName) + self.scratchDb.dropTable(cptTableName) + + self.scratchDb.setImportedThisDvoTable(cpxPath, region) + + return True Index: /trunk/ippToPsps/jython/dvoforcedobjects.py =================================================================== --- /trunk/ippToPsps/jython/dvoforcedobjects.py (revision 37551) +++ /trunk/ippToPsps/jython/dvoforcedobjects.py (revision 37551) @@ -0,0 +1,67 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from dvo import Dvo + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoForcedObjects(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): + + super(DvoForcedObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + # declare DVO file types of interest + self.ingestFileTypes = ['cpt', 'cps', 'cpy'] + self.loadPhotcodes() + + ''' + ingest forced object data into MySQL database using the native DVO program dvopsps + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def nativeIngestRegion(self, region): + + # drop cpt/cps/cpy/object table? + # create detections table? + + # TODO path to DVO prog hardcoded temporarily + # TODO this needs to be modified for forced objects + cmd = "dvopsps objects" + cmd += " -dbhost " + self.scratchDb.dbHost + cmd += " -dbname " + self.scratchDb.dbName + cmd += " -dbuser " + self.scratchDb.dbUser + cmd += " -dbpass " + self.scratchDb.dbPass + cmd += " -D CATDIR " + self.skychunk.dvoLocation + cmd += " -cpt " + region + + if self.skychunk.parallel: + if self.scratchDb.dbHost == "localhost": + self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb") + raise + + cmd += " -parallel" + + self.logger.infoPair("Running dvopsps", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", region) + raise + + # update lists after attempted sync + # self.printSummary() + + return True Index: /trunk/ippToPsps/jython/forcedobjectbatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedobjectbatch.py (revision 37550) +++ /trunk/ippToPsps/jython/forcedobjectbatch.py (revision 37551) @@ -0,0 +1,360 @@ +#!/usr/bin/env jython + +import os.path +import sys + +import stilts +from java.lang import * +from java.sql import * + +from xml.etree.ElementTree import ElementTree, Element, tostring + +from batch import Batch +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb +from dvoforcedobjects import DvoForcedObjects # need one specific for cpy ? +from sqlUtility import sqlUtility + +import logging.config + +''' +ForcedObjectBatch class + +This class, a sub-class of Batch, processes individual DVO region file pairs (cpt, cpy, and cps files). The cpt file contains a list of all objects in that region, the cps file contains the magnitudes and so has a row per filter per object (so, generally, has rows = 5 x number of objects). The Photcode.dat table us used to figure out the number and order of filters as they appear in the cps file. + +''' +class ForcedMeanObjectBatch(Batch): + + ''' + Constructor + ''' + def __init__(self, + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID): + + super(ForcedMeanObjectBatch, self).__init__( + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID, + "FO", + None) + + try: + self.dvoForcedObjects = DvoForcedObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) + except: + self.logger.errorPair("Unable to create instance of", "DvoForcedObjects") + raise + + # create an output filename, which is {dvoINDEX}.FITS + self.outputFitsFile = "%08d.FITS" % self.id + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + + # dump stuff to log + self.logger.infoPair("DVO INDEX", "%d" % self.id) + + ''' + Overriden from batch base-class. We import from DVO directly for forced mean objects + ''' + def importIppTables(self, columns="*", filter=""): + + self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id) + self.ippToPspsDb.insertForcedMeanObjectMeta(self.batchID, self.region) + self.dvoForcedObjects.nativeIngestRegion(self.region) + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + + if not self.scratchDb.tableExists(cptTableName): + self.logger.infoPair("can't find in scratch db:","cpt table") + return False + + cpyTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpy") + + if not self.scratchDb.tableExists(cpyTableName): + self.logger.infoPair("can't find in scratch db:","cpy table") + return False + + return True + + ''' + Applies indexes to the PSPS tables + ''' + def alterPspsTables(self): + + return True + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + # since dvopsps is now used, no action is needed here + # self.logger.infoPair("Creating indexes on", "IPP tables") + + return True + + ''' + Inserts stuff for all mags + ''' + def updateForcedMeanObjectFromCps(self, cpsTable): + + # list of all filters PSPS is interested in + # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere + interestedFilters = ['g', 'r', 'i', 'z', 'y'] + + filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters) + + # get a count of the available filters + filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable() + + self.logger.infoPair("Available filters in Photcodes", filters) + + # the 'code' now defines the order in the cps file that the mags are listed for a given filter + self.logger.infoPair("Adding magnitudes from", "cps table") + for filter in filters: + + filterID = self.scratchDb.getFilterID(filter[1]) + + # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000) + # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags + # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!) + + # set the MeanObject fields based largely on dvopsps cps fields: + + # XXX EAM 20140724 : filterCount is meant to match + # Nsecfilt, but is potentially not determined correctly. + # use a call to a dvo-native command which knows how to + # find Nsecfilt (or save in the db with dvopsps) + + # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting? + # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter + # " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + + sql = "UPDATE ForcedMeanObject JOIN \ + " + cpsTable + " AS cps ON (cps.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + SET \ + ForcedMeanObject." + filter[1] + "nTotal = NWARP \ + ,ForcedMeanObject." + filter[1] + "nIncPSFMag = NUSED_WRP \ + ,ForcedMeanObject." + filter[1] + "nIncKronMag = NUSED_KRON_WRP \ + ,ForcedMeanObject." + filter[1] + "nIncApMag = NUSED_AP_WRP \ + ,ForcedMeanObject." + filter[1] + "PSFMag = MAG_PSF_WRP \ + ,ForcedMeanObject." + filter[1] + "PSFMagErr = MAG_PSF_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "PSFMagStd = MAG_PSF_WRP_STD \ + ,ForcedMeanObject." + filter[1] + "KronMag = MAG_KRON_WRP \ + ,ForcedMeanObject." + filter[1] + "KronMagErr = MAG_KRON_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "KronMagStd = MAG_KRON_WRP_STD \ + ,ForcedMeanObject." + filter[1] + "ApMag = MAG_AP_WRP \ + ,ForcedMeanObject." + filter[1] + "ApMagErr = MAG_AP_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "ApMagStd = MAG_AP_WRP_STD \ + ,ForcedMeanObject." + filter[1] + "Flags = FLAGS " + + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed update MeanObject", sql) + raise + + # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12) + # XXX EAM 20140724 : keep this cut? + cut = 0.5 + self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut) + for filter in filters: + + sql = "UPDATE ForcedMeanObject \ + SET " + filter[1] + "PSFMagErr = null \ + WHERE " + filter[1] + "PSFMagErr > " + str(cut) + self.scratchDb.execute(sql) + + self.logger.infoPair("Calculating nDetections from", "n[filters]") + for filter in filters: + # now do a sum of n[filters], but do not include the ones with -999 + sql = "UPDATE ObjectThin " + sql += "SET nDetections = nDetections + n" + filter[1] + sql += " WHERE n" + filter[1] + " != -999" + self.scratchDb.execute(sql) + + ''' + Inserts stuff for all mags from the cpy files + ''' + def updateForcedMeanObjectFromCpy(self, cpyTable): + + # list of all filters PSPS is interested in + # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere + interestedFilters = ['g', 'r', 'i', 'z', 'y'] + + filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters) + + # get a count of the available filters + filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable() + + self.logger.infoPair("Available filters in Photcodes", filters) + + # the 'code' now defines the order in the cps file that the mags are listed for a given filter + self.logger.infoPair("Adding magnitudes from", "cpy table") + for filter in filters: + + filterID = self.scratchDb.getFilterID(filter[1]) + + # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000) + # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags + # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!) + + # set the MeanObject fields based largely on dvopsps cps fields: + + # XXX EAM 20140724 : filterCount is meant to match + # Nsecfilt, but is potentially not determined correctly. + # use a call to a dvo-native command which knows how to + # find Nsecfilt (or save in the db with dvopsps) + + # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting? + # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter + # " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + + sql = "UPDATE ForcedMeanObject JOIN \ + " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + SET \ + ForcedMeanObject." + filter[1] + "nIncR5 = NMEAS \ + ,ForcedMeanObject." + filter[1] + "nIncR6 = NMEAS \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR5 = FLUX_AP_R5 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR5Err = FLUX_ERR_AP_R5 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR5Std = FLUX_STD_AP_R5 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR5Fill = FLUX_FIL_AP_R5 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR6 = FLUX_AP_R6 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR6Err = FLUX_ERR_AP_R6 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR6Std = FLUX_STD_AP_R6 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR6Fill = FLUX_FIL_AP_R6 \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearX11 = X11_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearX12 = X12_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearX22 = X22_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearE1 = E1_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearE2 = E2_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearX11 = X11_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearX12 = X12_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearX22 = X22_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearE1 = E1_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearE2 = E2_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearX11 = X11_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearX12 = X12_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearX22 = X22_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearE1 = E1_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearE2 = E2_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearX11 = X11_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearX12 = X12_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearX22 = X22_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearE1 = E1_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearE2 = E2_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "Gamma = GAMMA \ + ,ForcedMeanObject." + filter[1] + "E1 = E1 \ + ,ForcedMeanObject." + filter[1] + "E2 = E2 " + + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed update on ForcedMeanObject using cpy table", sql) + raise + + + ''' + Populates the ForcedMeanObject table + ''' + def populateForcedMeanObjectTable(self): + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") + cpyTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpy") + + + self.logger.info("Populating ForcedMeanObject") + self.logger.info("Inserting objects from cpt file") + + # note "dec" is a reserved word in MySQL + # XXX EAM 20140724 : do not use INGORE unless we discover unavoidable problems... + # INSERT IGNORE INTO ObjectThin + + sqlLine = sqlUtility("INSERT INTO ForcedMeanObject (") + + sqlLine.group("objID", "EXT_ID") + sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly + sqlLine.group("batchID", "'" + str(self.batchID) + "'") + sqlLine.group("nDetections", "'0'") + sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't populate Object table from cpt", sql) + return False + + # add row count columns so we can perform joins to get colors + self.logger.infoPair("Adding 'row' columns to", "Object and cps tables") + self.scratchDb.addRowCountColumn("ForcedMeanObject", "row") + self.scratchDb.addRowCountColumn(cpsTableName, "row") + + + self.logger.infoPair("update ForcedMeanObject from ","cps table") + + self.updateForcedMeanObjectFromCps(cpsTableName) + + self.logger.infoPair("update ForcedMeanObject from ","cpy table") + + self.updateForcedMeanObjectFromCpy(cpyTableName) + + + self.logger.infoPair("Dropping row column from", "ForcedMeanObject table") + self.scratchDb.dropColumn("ForcedMeanObject", "row") + + ##self.logger.infoPair("Purging from scratch Db", self.region + " region") + + ##Don't do this till after MeanObject + ##self.dvoForcedObjects.purgeRegion(self.region) + + self.setMinMaxObjID(["ForcedMeanObject"]) + + return True + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + + if not self.populateForcedMeanObjectTable(): return False + + # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number + self.logger.infoPair("Forcing uniqueness on", "objID in ForcedMeanObject table") + rowCountBefore = self.scratchDb.getRowCount("ForcedMeanObject") + + # XXX EAM : note that in mysql versions later than 5.1, this fails + # unless the following is called first: + # set session old_alter_table=1 + # follow the command with + # set session old_alter_table=0 + # OF COURSE, this fails for mysql version < 5.5... + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=1") + + self.scratchDb.execute("ALTER IGNORE TABLE ForcedMeanObject ADD UNIQUE INDEX(objID)") + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=0") + + rowCountAfter = self.scratchDb.getRowCount("ForcedMeanObject") + self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + + self.dvoForcedObjects.purgeRegion(self.region) + + #this is abuse of something but this is how I get the object batches to crash to further investigate them + +# rowCountAfter = self.scratchDb.getRowCount("Object") + return True +# return False Index: /trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 37550) +++ /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 37551) @@ -0,0 +1,343 @@ +#!/usr/bin/env jython + +import os.path +import sys +import glob +import time +import stilts + +from java.lang import * +from java.sql import * + +from java.lang import Math +from org.apache.commons.math.special import Erf + +from xml.etree.ElementTree import ElementTree, Element, tostring + +from batch import Batch +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb +from sqlUtility import sqlUtility + +import logging.config + + +''' +ForcedWarpBatch class + +This class, a sub-class of Batch, processes single exposures in the form of IPP smf files + +''' +class ForcedWarpBatch(Batch): + + ''' + Constructor + ''' + def __init__(self, + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + forcedWarpID, + batchID): + + super(ForcedWarpBatch, self).__init__( + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + forcedWarpID, + batchID, + "P2", + gpc1Db.getForcedWarpStageSmf(forcedWarpID) + + # get camera meta data + meta = self.gpc1Db.getForcedWarpStageMeta(self.id) + if not meta: + self.logger.errorPair("Could not get", "forced warp metadata") + raise + + self.expID = meta[0]; + self.expName = meta[1]; + self.distGroup = meta[2]; + self.analysisVer = meta[3]; + self.bias = meta[4]; + self.biasScat = meta[5]; + + if not self.analysisVer: self.analysisVer = -999 + + # create an output filename, which is {expID}.FITS + self.outputFitsFile = "%08d.FITS" % self.expID + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + self.dropTableVerbose("ForcedWarpMeasurement") + self.dropTableVerbose("ForcedWarpExtended") + self.dropTableVerbose("ForcedWarpMeta") + self.dropTableVerbose("ForcedWarpToImage") + if not self.config.retry: + self.scratchDb.dropTable("ForcedWarpMeasurement") + self.scratchDb.dropTable("ForcedWarpExtended") + self.scratchDb.dropTable("ForcedWarpMeta") + self.scratchDb.dropTable("ForcedWarpToImage") + self.historyModNum = "0" + + # get a few primary header values. if in test mode, then use defaults + if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL": + self.logger.errorPair("Could not get", "MJD-OBS") + raise + + + if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL": + self.logger.errorPair("Could not get", "EXPTIME") + raise + + if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL": + self.logger.errorPair("Could not get", "FILTERID") + raise + + # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2 + self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0) + + # set up some defauts + self.calibModNum = 0 + self.totalNumPhotoRef = 0 + + + self.filter = self.header['FILTERID'][0:1] + self.filterID = self.scratchDb.getFilterID(self.filter) + + # insert what we know about this stack batch into the stack table + self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter) + + # dump stuff to log + self.logger.infoPair("Cam ID", "%d" % self.id) + self.logger.infoPair("Exp ID", "%d" % self.expID) + self.logger.infoPair("Exp name", "%s" % self.expName) + self.logger.infoPair("Distribution group", "%s" % self.distGroup) + + + + + ''' + Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header + ''' + def populateForcedWarpMeta(self): + self.logger.infoPair("Processing table", "ForcedWarpMeta") + + photoCalID = str(self.scratchDb.getPhotoCalID(forcedWarpID)) + + + sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta (") + #batchID (below) + #surveyID (below) + #filterdID (below) + sqlLine.group("skyCellID", str(self.skycellID)) + sqlLine.group("photoCalID", photoCalID) + sqlLine.group("magSat", self.getKeyFloat(header, "%.8f", 'FSATUR')) + sqlLine.group("analVer", self.analysisVer); + sqlLine.group("expTime", self.getKeyFloat(self.header, "%.8f", 'EXPREQ')); + sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT')) + sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR')) + sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')); + sqlLine.group("nAstroRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("nPhotoRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("psfModelID", self.getKeyValue(header, 'PSFMODEL')) + sqlLine.group("psfFwhm_mean", self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')) + sqlLine.group("psfFwhm_max", self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ')) + sqlLine.group("photoZero", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) + #photoColor -- how do I set this? it's also not set in stack meta + sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1')); + sqlLine.group("ctype2", self.getKeyValue(self.header, 'CTYPE2')); + sqlLine.group("crval1", self.getKeyFloat(self.header, "%.8f", 'CRVAL1')); + sqlLine.group("crval2", self.getKeyFloat(self.header, "%.8f", 'CRVAL2')); + sqlLine.group("crpix1", self.getKeyFloat(self.header, "%.8f", 'CRPIX1')); + sqlLine.group("crpix2", self.getKeyFloat(self.header, "%.8f", 'CRPIX2')); + sqlLine.group("cdelt1", self.getKeyFloat(self.header, "%.8e", 'CDELT1')); + sqlLine.group("cdelt2", self.getKeyFloat(self.header, "%.8e", 'CDELT2')); + sqlLine.group("pc001001", self.getKeyFloat(self.header, "%.8e", 'PC001001')); + sqlLine.group("pc001002", self.getKeyFloat(self.header, "%.8e", 'PC001002')); + sqlLine.group("pc002001", self.getKeyFloat(self.header, "%.8e", 'PC002001')); + sqlLine.group("pc002002", self.getKeyFloat(self.header, "%.8e", 'PC002002')); + + + sql = sqlLine.make(") VALUES ( ", ")") + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql: ', sql) + raise + + self.scratchDb.updateAllRows("ForcedWarpMeta", "batchID", str(self.batchID)) + self.scratchDb.updateAllRows("ForcedWarpMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("ForcedWarpMeta", self.filter) + self.scratchDb.updateAllRows("ForcedWarpMeta", "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows("ForcedWarpMeta", "dataRelease", str(self.skychunk.dataRelease)) + self.tablesToExport.append("ForcedWarpMeta") + + ''' + Populates the ForcedWarpToImage table + ''' + def populateForcedWarpToImage(self): + + self.logger.infoPair("Procesing table", "ForcedWarpToImage") + + for filter in self.filters: + + forcedwarpID = self.forcedwarpID + if forcedwarpID > 0: + + imageIDs = self.gpc1Db.getImageIDsForThisForcedWarpID(forcedwarpID) + + for imageID in imageIDs: + + sql = "INSERT INTO ForcedWarpToImage (forcedwarpID, imageID) \ + VALUES (\ + " + str(forcedwarpID) + ", " + imageID + ")" + self.scratchDb.execute(sql) + + # now update StackMeta with correct number of inputs + + self.tablesToExport.append("ForcedWarpToImage") + + + + + + ''' + Populates the ForcedWarpMeasurement table + ''' + def populateForcedWarpMeasurementTable(self, results): + + pspsTableName = "ForcedWarpMeasurement" + ippTableName = "ForcedWarpMeasurement_psf" + + + self.logger.infoPair("Procesing table", "ForcedWarpMeasurement") + + # insert the per-object data (IDs, random stack ID, dataRelease, etc) + self.selectDvoObjIDs() + + self.generateRandomIDs() + + # add indexes ForcedWarpMeasurement + self.scratchDb.createIndex("ForcedWarpMeasurement", "objID") + + + + + + + + + + self.tablesToExport.append("ForcedWarpMeasurement") + + + ''' + Populates the ForcedWarpExtended table + ''' + def populateForcedWarpExtendedTable(self, results): + + pspsTableName = "ForcedWarpExtended" + ippTableName = "ForcedWarpExtended_psf" + + + self.tablesToExport.append("ForcedWarpExtended") + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + self.logger.infoPair("starting","populatePspsTables"); + self.skipBatch = False + # each of the "populate*" methods below add their tables to the list: + self.tablesToExport=[] + + #this should be done + self.populateForcedWarpMeta() + + self.populateForcedWarpMeasurement() + self.populateForcedWarpExtended() + self.populateForcedWarpToImage() + + self.setMinMaxObjID(["ForcedWarpMeasurement"]) + + self.logger.infoPair("finishing","populatePspsTables"); + return True + + + + ''' + This function reads the cmf/smf file and loads it into the database as its own table + ''' + def importIppTables(self, columns="*", tableRE=""): + + if self.config.retry: return True + self.logger.infoPair("Importing FW tables with table match expression: ", tableRE) + fileName = self.fits.getPath() + + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + for table in tables: + + match = re.match(tableRE, table.name) + if not match: continue + self.logger.infoPair("Reading IPP table", table.name) + table = stilts.tpipe(table, cmd='addcol table_index $0') + table = stilts.tpipe(table, cmd='explodeall') + + # drop any previous tables before import + self.scratchDb.dropTable(table.name) + + # IPP FITS files are littered with infinities, so remove these + self.logger.info("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + try: + table.write(self.scratchDb.url + '#' + table.name) + count = count + 1 + except: + self.logger.exception("Problem writing table '" + table.name + "' to the database") + + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables() + + return True + + + + + + + + + + def generateRandomIDs(self): + sql = "UPDATE ForcedWarpMeasurement set randomWarpID = FLOOR(RAND()*9223372036854775807)"; + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + return + + + + + + + + + + def exportPspsTablesToFits(self, regex="(.*)"): + return super(DetectionBatch, self).exportPspsTablesToFits("(ForcedWarp.*)") Index: /trunk/ippToPsps/jython/gpc1db.py =================================================================== --- /trunk/ippToPsps/jython/gpc1db.py (revision 37550) +++ /trunk/ippToPsps/jython/gpc1db.py (revision 37551) @@ -116,5 +116,26 @@ stage = "forcedwarp" - sql = "SELECT differentStuff from Stufftable" + sql = "SELECT DISTINCT sky_id, radeg, decdeg FROM fullForceInput \ + JOIN addRun ON(fullForceInput.ff_id = addRun.stage_id, fullForceInput.warp_id = addRun.stage_extra1) \ + JOIN minidvodbRun USING(minidvodb_name) \ + JOIN minidvodbProcessed USING(minidvodb_id) \ + JOIN mergedvodbRun USING(minidvodb_id) \ + JOIN mergedvodbProcessed USING (merge_id) \ + JOIN fullForceRun USING(ff_id) \ + JOIN skycalRun using (skycal_id) + JOIN stackRun USING(stack_id) \ + JOIN stackSumSkyfile using (stack_id) \ + JOIN skycell USING(skycell_id) \ + WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND minidvodbRun.state = 'merged' \ + AND minidvodbProcessed.fault = 0 \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbProcessed.fault = 0 \ + AND addRun.stage = '" + stage + "' \ + AND addRun.state = 'full' \ + AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ + AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + AND mjd_obs >= (to_days('" + tstart + "')-678941) \ + AND mjd_obs <= (to_days('" + tend + "') - 678941) " try: @@ -165,4 +186,9 @@ return imageIDs + + + + + ''' Index: /trunk/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /trunk/ippToPsps/jython/ipptopspsdb.py (revision 37550) +++ /trunk/ippToPsps/jython/ipptopspsdb.py (revision 37551) @@ -756,4 +756,81 @@ self.execute(sql) + + ''' + Inserts some forced detection metadata for this batch ID + ''' + def insertForcedWarpMeta(self, batchID, expID, filter): + + sql = "INSERT INTO forcedwarp ( \ + batch_id \ + ,exp_id \ + ,filter \ + ) VALUES ( \ + " + str(batchID) + " \ + ," + str(expID) + " \ + ,'" + filter + "' \ + )" + + self.execute(sql) + + + + ''' + Inserts some forcedobject metadata for this batch ID + ''' + def insertForcedObjectMeta(self, batchID, region): + + sql = "INSERT INTO forcedobject ( \ + batch_id \ + ,region \ + ) VALUES ( \ + " + str(batchID) + " \ + ,'" + region + "' \ + )" + + self.execute(sql) + + + + + ''' + Inserts some diff metadata for this batch ID + ''' + def insertDiffMeta(self, batchID, filter, diffType): + + sql = "INSERT INTO stack ( \ + batch_id \ + ,filter \ + ,diff_type \ + ) VALUES ( \ + " + str(batchID) + " \ + ,'" + filter + "' \ + ,'" + diffType + "')" + + self.execute(sql) + + + + + ''' + Inserts some diffobject metadata for this batch ID + ''' + def insertDiffObjectMeta(self, batchID, region): + + sql = "INSERT INTO diffobject ( \ + batch_id \ + ,region \ + ) VALUES ( \ + " + str(batchID) + " \ + ,'" + region + "' \ + )" + + self.execute(sql) + + + + + + ''' Resets a batch ready for re-loading to the datastore Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 37550) +++ /trunk/ippTools/share/Makefile.am (revision 37551) @@ -8,4 +8,8 @@ addtool_find_cam_id_dvo.sql \ addtool_find_cam_id.sql \ + addtool_find_diff_id.sql \ + addtool_find_diff_id_dvo.sql \ + addtool_find_ff_id.sql \ + addtool_find_ff_id_dvo.sql \ addtool_find_skycal_id_dvo.sql \ addtool_find_skycal_id.sql \ @@ -16,4 +20,6 @@ addtool_find_pendingexp.sql \ addtool_find_pendingexp_cam.sql \ + addtool_find_pendingexp_diff.sql \ + addtool_find_pendingexp_ff.sql \ addtool_find_pendingexp_skycal.sql \ addtool_find_pendingexp_stack.sql \ @@ -21,4 +27,6 @@ addtool_find_pendingmergeprocess.sql \ addtool_find_processedexp_cam.sql \ + addtool_find_processedexp_diff.sql \ + addtool_find_processedexp_ff.sql \ addtool_find_processedexp_skycal.sql \ addtool_find_processedexp_stack.sql \ @@ -29,4 +37,6 @@ addtool_pendingcleanuprun.sql \ addtool_queue_cam_id.sql \ + addtool_queue_diff_id.sql \ + addtool_queue_ff_id.sql \ addtool_queue_stack_id.sql \ addtool_queue_sky_id_multi.sql \ @@ -35,4 +45,6 @@ addtool_revertminidvodbprocessed.sql \ addtool_revertprocessedexp_cam.sql \ + addtool_revertprocessedexp_diff.sql \ + addtool_revertprocessedexp_ff.sql \ addtool_revertprocessedexp_skycal.sql \ addtool_revertprocessedexp_stack.sql \ Index: /trunk/ippTools/share/addtool_find_diff_id.sql =================================================================== --- /trunk/ippTools/share/addtool_find_diff_id.sql (revision 37551) +++ /trunk/ippTools/share/addtool_find_diff_id.sql (revision 37551) @@ -0,0 +1,22 @@ +SELECT + diffRun.*, diffInputSkyfile.diff_skyfile_id +FROM + diffInputSkyfile +JOIN diffSkyfile using (diff_id, skycell_id) +JOIN diffRun using (diff_id) + +LEFT JOIN (SELECT diff_id AS added_diff_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN diffRun on (diff_id = stage_id and addRun.stage = 'diff') + ) as foo + ON diff_id = added_diff_id + -- AND stage = 'diff' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + diffRun.state = 'full' + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND added_diff_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /trunk/ippTools/share/addtool_find_diff_id_dvo.sql =================================================================== --- /trunk/ippTools/share/addtool_find_diff_id_dvo.sql (revision 37551) +++ /trunk/ippTools/share/addtool_find_diff_id_dvo.sql (revision 37551) @@ -0,0 +1,15 @@ +SELECT diffRun.*, diffInputSkyfile.diff_skyfile_id +FROM + diffInputSkyfile +JOIN diffSkyfile using (diff_id, skycell_id) +JOIN diffRun using (diff_id) + + +diffRun JOIN diffSkyfile using (diff_id) + +WHERE diffRun.state = 'full' and diffSkyfile.quality = 0 + AND diffRun.diff_id NOT IN (SELECT diffRun.diff_id + FROM addRun + JOIN diffRun on (diffRun.diff_id = addRun.stage_id and addRun.stage = 'diff') JOIN diffSkyfile on (diffRun.diff_id = diffSkyfile.diff_id) + WHERE addRun.stage = 'diff' AND %s + ) Index: /trunk/ippTools/share/addtool_find_ff_id.sql =================================================================== --- /trunk/ippTools/share/addtool_find_ff_id.sql (revision 37551) +++ /trunk/ippTools/share/addtool_find_ff_id.sql (revision 37551) @@ -0,0 +1,21 @@ +SELECT + fullForceRun.*, fullForceResult.warp_id +FROM + fullForceRun +JOIN fullForceResult using (ff_id) + +LEFT JOIN (SELECT ff_id AS added_ff_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN fullForceRun on (ff_id = stage_id and addRun.stage = 'fullforce') + ) as foo + ON ff_id = added_ff_id + --AND stage = 'skycal' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + fullForceRun.state = 'full' + AND fullForceResult.quality = 0 + AND fullForceResult.fault = 0 + AND added_ff_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /trunk/ippTools/share/addtool_find_ff_id_dvo.sql =================================================================== --- /trunk/ippTools/share/addtool_find_ff_id_dvo.sql (revision 37551) +++ /trunk/ippTools/share/addtool_find_ff_id_dvo.sql (revision 37551) @@ -0,0 +1,8 @@ +SELECT fullForceRun.*, fullForceResult.warp_id FROM fullForceRun JOIN fullForceResult using (ff_id) + +WHERE fullForceRun.state = 'full' and fullForceResult.quality = 0 + AND fullForceRun.ff_id NOT IN (SELECT fullForceRun.ff_id + FROM addRun + JOIN fullForceRun on (fullForceRun.ff_id = addRun.stage_id and addRun.stage = 'fullforce') JOIN fullForceResult on (fullForceRun.ff_id = fullForceResult.ff_id) + WHERE addRun.stage = 'fullforce' AND %s + ) Index: /trunk/ippTools/share/addtool_find_pendingexp_diff.sql =================================================================== --- /trunk/ippTools/share/addtool_find_pendingexp_diff.sql (revision 37551) +++ /trunk/ippTools/share/addtool_find_pendingexp_diff.sql (revision 37551) @@ -0,0 +1,21 @@ + SELECT + addRun.*, + diffSkyfile.path_base as stageroot, + "NULL", + "NULL" +FROM addRun +JOIN diffInputSkyfile + ON (diff_id = stage_id and diff_skyfile_id = stage_extra1 and stage = "diff") +JOIN diffSkyfile + USING (diff_id, skycell_id) +JOIN diffRun + using(diff_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + diffRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL Index: /trunk/ippTools/share/addtool_find_pendingexp_ff.sql =================================================================== --- /trunk/ippTools/share/addtool_find_pendingexp_ff.sql (revision 37551) +++ /trunk/ippTools/share/addtool_find_pendingexp_ff.sql (revision 37551) @@ -0,0 +1,19 @@ + SELECT + addRun.*, + fullForceResult.path_base as stageroot, + "NULL", + "NULL" +FROM addRun +JOIN fullForceResult + ON (ff_id = stage_id and warp_id = stage_extra1 and stage = "fullforce") +JOIN fullForceRun + using(ff_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + fullForceRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL Index: /trunk/ippTools/share/addtool_find_processedexp_diff.sql =================================================================== --- /trunk/ippTools/share/addtool_find_processedexp_diff.sql (revision 37551) +++ /trunk/ippTools/share/addtool_find_processedexp_diff.sql (revision 37551) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /trunk/ippTools/share/addtool_find_processedexp_ff.sql =================================================================== --- /trunk/ippTools/share/addtool_find_processedexp_ff.sql (revision 37551) +++ /trunk/ippTools/share/addtool_find_processedexp_ff.sql (revision 37551) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /trunk/ippTools/share/addtool_queue_diff_id.sql =================================================================== --- /trunk/ippTools/share/addtool_queue_diff_id.sql (revision 37551) +++ /trunk/ippTools/share/addtool_queue_diff_id.sql (revision 37551) @@ -0,0 +1,24 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'diff', -- stage + diff_id, -- stage_id + diff_skyfile_id, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM diffInputSkyfile + JOIN diffSkyfile using (diff_id, skycell_id) + JOIN diffRun using (diff_id) + WHERE + diffRun.state = 'full' AND + AND diffRun.diff_id = %lld Index: /trunk/ippTools/share/addtool_queue_ff_id.sql =================================================================== --- /trunk/ippTools/share/addtool_queue_ff_id.sql (revision 37551) +++ /trunk/ippTools/share/addtool_queue_ff_id.sql (revision 37551) @@ -0,0 +1,23 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'fullforce', -- stage + ff_id, -- stage_id + warp_id, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM fullForceResult + JOIN fullForceRun using (ff_id) + WHERE + fullForceRun.state = 'full' + AND fullForceRun.ff_id = %lld Index: /trunk/ippTools/share/addtool_revertprocessedexp_diff.sql =================================================================== --- /trunk/ippTools/share/addtool_revertprocessedexp_diff.sql (revision 37551) +++ /trunk/ippTools/share/addtool_revertprocessedexp_diff.sql (revision 37551) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'diff' Index: /trunk/ippTools/share/addtool_revertprocessedexp_ff.sql =================================================================== --- /trunk/ippTools/share/addtool_revertprocessedexp_ff.sql (revision 37551) +++ /trunk/ippTools/share/addtool_revertprocessedexp_ff.sql (revision 37551) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'fullforce' Index: /trunk/ippTools/src/addtool.c =================================================================== --- /trunk/ippTools/src/addtool.c (revision 37550) +++ /trunk/ippTools/src/addtool.c (revision 37551) @@ -122,4 +122,7 @@ PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); if (strcmp(stage, "cam")== 0) { @@ -146,5 +149,17 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "skycalRun.reduction", "=="); } - + if (strcmp(stage, "diff")==0) { + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); //define using skycalRun label + pxAddLabelSearchArgs (config, where, "-data_group","diffRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "diffRun.reduction", "=="); + } + if (strcmp(stage, "fullforce")==0) { + pxAddLabelSearchArgs (config, where, "-label", "fullForceRun.label", "=="); //define using skycalRun label + pxAddLabelSearchArgs (config, where, "-data_group","fullForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "fullForceRun.reduction", "=="); + } + + + if (!psListLength(where->list)) { psFree(where); @@ -258,5 +273,37 @@ } - + if (strcmp(stage,"diff") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_diff_id_dvo.sql\n%s\n", dvodb,stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_diff_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_diff_id.sql\n%s\n",stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_diff_id.sql"); + // inherit dvodb from skycalRun, avoid matching NULL + psStringAppend(&dvodb_string, "(diffRun.dvodb IS NOT NULL AND previous_dvodb = diffRun.dvodb)"); + // this is silly, there is no dvodb in skycalRun...? + } + } + + if (strcmp(stage,"fullforce") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_ff_id_dvo.sql\n%s\n", dvodb,stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_ff_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_ff_id.sql\n%s\n",stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_ff_id.sql"); + // inherit dvodb from skycalRun, avoid matching NULL + psStringAppend(&dvodb_string, "(fullForceRun.dvodb IS NOT NULL AND previous_dvodb = fullForceRun.dvodb)"); + // this is silly, there is no dvodb in skycalRun...? + } + } @@ -324,4 +371,15 @@ psStringAppend(&query, " GROUP BY skycal_id, sky_id, stack_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there } + //needs to be checked HAF xxx + if (strcmp(stage,"diff") == 0) { + psStringAppend(&query, " GROUP BY diff_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there + } + //needs to be checked HAF xxx + if (strcmp(stage,"fullforce") == 0) { + psStringAppend(&query, " GROUP BY ff_id, skycal_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there + } + + + psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); @@ -476,4 +534,57 @@ } + if (strcmp(stage,"diff") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + diffRunRow *row = diffRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, diff_id %" PRId64, row->label, row->diff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, diff_id %" PRId64, row->label, row->diff_id); + psFree(output); + return false; + } + + psFree(row); + } + } + + if (strcmp(stage,"fullforce") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + fullForceRunRow *row = fullForceRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->ff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->ff_id); + psFree(output); + return false; + } + + psFree(row); + } + } @@ -672,4 +783,92 @@ } + if (strcmp(stage,"diff") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + diffRunRow *row = diffRunObjectFromMetadata(md); + stage_id = row->diff_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + 0, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + } + + if (strcmp(stage,"fullforce") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + fullForceRunRow *row = fullForceRunObjectFromMetadata(md); + stage_id = row->ff_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + 0, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + } + + + @@ -695,4 +894,7 @@ PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); pxcamGetSearchArgs (config, where); // most search arguments based on camera @@ -709,14 +911,20 @@ psString query = NULL; if (strcmp(stage, "cam")==0) { - query = psStringCopy("UPDATE addRun JOIN camRun on cam_id = stage_id JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + query = psStringCopy("UPDATE addRun JOIN camRun on (cam_id = stage_id and stage = 'cam') JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); } if (strcmp(stage, "stack")==0) { - query = psStringCopy("UPDATE addRun JOIN stackRun on stack_id = stage_id"); + query = psStringCopy("UPDATE addRun JOIN stackRun on (stack_id = stage_id and stage = 'stack')"); } if (strcmp(stage, "staticsky")==0) { - query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); + query = psStringCopy("UPDATE addRun JOIN staticskyRun on (sky_id = stage_id and stage = 'staticsky')"); } if (strcmp(stage, "skycal")==0) { - query = psStringCopy("UPDATE addRun JOIN skycalRun on skycal_id = stage_id"); + query = psStringCopy("UPDATE addRun JOIN skycalRun on (skycal_id = stage_id and stage = 'skycal')"); + } + if (strcmp(stage, "diff")==0) { + query = psStringCopy("UPDATE addRun JOIN diffInputSkyfile on (diff_id = stage_id and diff_skyfile_id = stage_extra1 and stage = 'diff') JOIN diffSkyfile on (diff_id, skycell_id) JOIN diffRun using (diff_id)"); + } + if (strcmp(stage, "fullforce")==0) { + query = psStringCopy("UPDATE addRun JOIN fullForceResult on (ff_id = stage_id and warp_id = stage_extra1 and stage = 'fullforce') JOIN fullForceRun on (ff_id)"); } @@ -743,4 +951,6 @@ PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); pxcamGetSearchArgs (config, where); @@ -764,4 +974,10 @@ query = pxDataGet("addtool_find_pendingexp_skycal.sql"); } + if (strcmp(stage, "diff")==0) { + query = pxDataGet("addtool_find_pendingexp_diff.sql"); + } + if (strcmp(stage, "fullforce")==0) { + query = pxDataGet("addtool_find_pendingexp_ff.sql"); + } @@ -790,4 +1006,15 @@ psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1"); } + if (strcmp(stage, "diff") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "diff_id, stage_extra1"); + } + if (strcmp(stage, "fullforce") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "ff_id, stage_extra1"); + } + + + // treat limit == 0 as "no limit" if (limit) { @@ -994,4 +1221,9 @@ } else if (strcmp (stage,"skycal") == 0) { query = pxDataGet("addtool_find_processedexp_skycal.sql"); + } else if (strcmp (stage,"diff") == 0) { + query = pxDataGet("addtool_find_processedexp_diff.sql"); + } else if (strcmp (stage,"fullforce") == 0) { + query = pxDataGet("addtool_find_processedexp_ff.sql"); + } else { psFree(where); @@ -1112,4 +1344,10 @@ if (strcmp(stage, "skycal") == 0) { query = pxDataGet("addtool_revertprocessedexp_skycal.sql"); + } + if (strcmp(stage, "diff") == 0) { + query = pxDataGet("addtool_revertprocessedexp_diff.sql"); + } + if (strcmp(stage, "fullforce") == 0) { + query = pxDataGet("addtool_revertprocessedexp_ff.sql"); } Index: /trunk/ippTools/src/addtoolConfig.c =================================================================== --- /trunk/ippTools/src/addtoolConfig.c (revision 37550) +++ /trunk/ippTools/src/addtoolConfig.c (revision 37551) @@ -55,4 +55,6 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal_id", 0); pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by ff_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by camRun data_group", NULL); @@ -83,4 +85,6 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id",0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id", 0, "search by _ff_id",0); pxcamSetSearchArgs(updaterunArgs); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by addRun label", NULL); @@ -101,7 +105,9 @@ psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack_id", 0); - psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); - psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id",0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-ff_id", 0, "search by ff_id",0); pxcamSetSearchArgs(pendingexpArgs); psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); @@ -236,5 +242,5 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definebyquery", "create runs from cam/skycal/forcedwarp/diff stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-updaterun", "change add run properties", ADDTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-pendingexp", "show pending exps", ADDTOOL_MODE_PENDINGEXP, pendingexpArgs); Index: /trunk/ippTools/src/pxadd.c =================================================================== --- /trunk/ippTools/src/pxadd.c (revision 37550) +++ /trunk/ippTools/src/pxadd.c (revision 37551) @@ -164,4 +164,14 @@ psMemSetPersistent(query, true); } + else if (strcmp(stage,"diff") == 0) { + query = pxDataGet("addtool_queue_diff_id.sql"); + psMemSetPersistent(query, true); + } + else if (strcmp(stage,"fullforce") == 0) { + query = pxDataGet("addtool_queue_ff_id.sql"); + psMemSetPersistent(query, true); + } + + else {