Index: /tags/ipp-20120216/ippMonitor/def/lapExposures.d =================================================================== --- /tags/ipp-20120216/ippMonitor/def/lapExposures.d (revision 33341) +++ /tags/ipp-20120216/ippMonitor/def/lapExposures.d (revision 33342) @@ -6,54 +6,55 @@ UNRESTRICTED WHERE 0=1 -ARGS ARG1 rawImfile.exp_id=$lapExp.exp_id -ARGS ARG2 chipRun.chip_id=$lapExp.chip_id -ARGS ARG3 chipRun.chip_id=$lapExp.pair_id -ARGS ARG4 camRun.cam_id=$camRun1.cam_id -ARGS ARG5 camRun.cam_id=$camRun2.cam_id -ARGS ARG6 fakeRun.fake_id=$fakeRun1.fake_id -ARGS ARG7 fakeRun.fake_id=$fakeRun2.fake_id -ARGS ARG8 warpRun.warp_id=$warpRun1.warp_id -ARGS ARG9 warpRun.warp_id=$warpRun2.warp_id -ARGS ARG10 diffRun.diff_id=$diffRun.diff_id +ARGS ARG1 rawImfile.exp_id=$lapExp.exp_id +ARGS ARG2 chipRun.chip_id=$lapExp.chip_id +ARGS ARG3 chipRun.chip_id=$lapExp.pair_id +ARGS ARG4 camRun.cam_id=$camRun1.cam_id +ARGS ARG5 camRun.cam_id=$camRun2.cam_id +ARGS ARG6 fakeRun.fake_id=$fakeRun1.fake_id +ARGS ARG7 fakeRun.fake_id=$fakeRun2.fake_id +ARGS ARG8 warpRun.warp_id=$warpRun1.warp_id +ARGS ARG9 warpRun.warp_id=$warpRun2.warp_id +ARGS ARG10 diffRun.diff_id=$diffRun.diff_id -# field size format name show link to extras -FIELD lapRun.seq_id, 6, %d, seq_id -FIELD lapExp.lap_id, 6, %d, lap_id -FIELD lapExp.data_state, 6, %s, State (lapExp) -FIELD lapExp.exp_id, 6, %d, Exp ID, value, rawImfile.php, ARG1 -FIELD lapExp.chip_id, 6, %d, Chip1 (lapExp.chip_id), value, chipStageExp.php, ARG2 -FIELD chipRun1.state AS ch1state, 6, %s, Chip1 State -FIELD camRun1.cam_id AS cam_id_1, 6, %d, Cam1 ID, value, camStageExp.php, ARG4 -FIELD camRun1.state AS ca1state, 6, %s, Cam1 State -FIELD fakeRun1.fake_id AS fake_id_1, 6, %d, Fake1 ID, value, fakeStageExp.php, ARG6 -FIELD fakeRun1.state AS fa1state, 6, %s, Fake1 State -FIELD warpRun1.warp_id AS warp_id_1, 6, %d, Warp1 ID, value, warpStageExp.php, ARG8 -FIELD warpRun1.state AS wa1state, 6, %s, Warp1 State -FIELD lapExp.pair_id, 6, %d, Chip2 ID (lapExp.chip_id), value, chipStageExp.php, ARG3 -FIELD chipRun2.state AS ch2state, 6, %s, Chip2 State -FIELD camRun2.cam_id AS cam_id_2, 6, %d, Cam2 ID, value, camStageExp.php, ARG5 -FIELD camRun2.state AS ca2state, 6, %s, Cam2 State -FIELD fakeRun2.fake_id AS fake_id_2, 6, %d, Fake2 ID, value, fakeStageExp.php, ARG7 -FIELD fakeRun2.state AS fa2state, 6, %s, Fake2 State -FIELD warpRun2.warp_id AS warp_id_2, 6, %d, Warp2 ID, value, warpStageExp.php, ARG9 -FIELD warpRun2.state AS wa2state, 6, %s, Warp2 State -FIELD diffRun.diff_id, 6, %d, Diff ID, value, diffStageExp.php, ARG10 -FIELD diffRun.state AS diState, 6, %s, Diff State -FIELD lapRun.tess_id, 8, %s, tess_id -FIELD lapRun.projection_cell, 10, %s, ProjCell -FIELD lapRun.filter, 7, %s, Filter -FIELD lapRun.state, 15, %s, State (lapRun) -FIELD lapRun.label, 15, %s, Label -FIELD lapRun.dist_group, 15, %s, Dist Group -FIELD lapRun.registered, 19, %T, Registered -FIELD lapRun.fault, 3, %d, fault -FIELD lapRun.quick_sass_id, 6, %d, quick_sass_id -FIELD lapRun.final_sass_id, 6, %d, final_sass_id -FIELD lapExp.private, 2, %d, Private -FIELD lapExp.pairwise, 2, %d, Pairwise -FIELD lapExp.active, 2, %d, Active -FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.object, 8, %s, Object -FIELD rawExp.comment, 80, %s, Comment +# field size format name show link to extras +FIELD lapRun.seq_id, 6, %d, seq_id +FIELD lapExp.lap_id, 6, %d, lap_id +FIELD lapExp.data_state, 6, %s, State (lapExp) +FIELD lapExp.exp_id, 6, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD lapExp.chip_id, 6, %d, Chip1 (lapExp.chip_id), value, chipStageExp.php, ARG2 +FIELD chipRun1.state AS ch1state, 6, %s, Chip1 State +FIELD camRun1.cam_id AS cam_id_1, 6, %d, Cam1 ID, value, camStageExp.php, ARG4 +FIELD camRun1.state AS ca1state, 6, %s, Cam1 State +FIELD fakeRun1.fake_id AS fake_id_1, 6, %d, Fake1 ID, value, fakeStageExp.php, ARG6 +FIELD fakeRun1.state AS fa1state, 6, %s, Fake1 State +FIELD warpRun1.warp_id AS warp_id_1, 6, %d, Warp1 ID, value, warpStageExp.php, ARG8 +FIELD warpRun1.state AS wa1state, 6, %s, Warp1 State +FIELD lapExp.pair_id, 6, %d, Chip2 ID (lapExp.chip_id), value, chipStageExp.php, ARG3 +FIELD chipRun2.state AS ch2state, 6, %s, Chip2 State +FIELD camRun2.cam_id AS cam_id_2, 6, %d, Cam2 ID, value, camStageExp.php, ARG5 +FIELD camRun2.state AS ca2state, 6, %s, Cam2 State +FIELD fakeRun2.fake_id AS fake_id_2, 6, %d, Fake2 ID, value, fakeStageExp.php, ARG7 +FIELD fakeRun2.state AS fa2state, 6, %s, Fake2 State +FIELD warpRun2.warp_id AS warp_id_2, 6, %d, Warp2 ID, value, warpStageExp.php, ARG9 +FIELD warpRun2.state AS wa2state, 6, %s, Warp2 State +FIELD diffRun.diff_id, 6, %d, Diff ID, value, diffStageExp.php, ARG10 +FIELD diffRun.state AS diState, 6, %s, Diff State +FIELD lapRun.tess_id, 8, %s, tess_id +FIELD lapRun.projection_cell, 10, %s, ProjCell +FIELD diffInputSkyfile.skycell_id, 10, %s, Skycell ID +FIELD lapRun.filter, 7, %s, Filter +FIELD lapRun.state, 15, %s, State (lapRun) +FIELD lapRun.label, 15, %s, Label +FIELD lapRun.dist_group, 15, %s, Dist Group +FIELD lapRun.registered, 19, %T, Registered +FIELD lapRun.fault, 3, %d, fault +FIELD lapRun.quick_sass_id, 6, %d, quick_sass_id +FIELD lapRun.final_sass_id, 6, %d, final_sass_id +FIELD lapExp.private, 2, %d, Private +FIELD lapExp.pairwise, 2, %d, Pairwise +FIELD lapExp.active, 2, %d, Active +FIELD rawExp.dateobs, 19, %T, Date/Time +FIELD rawExp.object, 8, %s, Object +FIELD rawExp.comment, 80, %s, Comment # MODE summary Index: /tags/ipp-20120216/ippScripts/scripts/camera_exp.pl =================================================================== --- /tags/ipp-20120216/ippScripts/scripts/camera_exp.pl (revision 33341) +++ /tags/ipp-20120216/ippScripts/scripts/camera_exp.pl (revision 33342) @@ -156,4 +156,10 @@ { # Determine if FWHM is too large to bother continuing. my $maxFWHM = 0; + # XXX: temporary hack if this is a CNP exposure set maxFWHM to a large + # value. + # Before the next tag we will put this into a recipe + if ($outroot =~ /CNP/) { + $maxFWHM = 25; + } else { my $command = "$ppConfigDump -camera $camera -get-key MAX_ALLOWED_FWHM"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -168,4 +174,5 @@ $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM'); } + } if ($maxFWHM) { my $expFWHM; Index: /tags/ipp-20120216/ippToPsps/config/settings.xml =================================================================== --- /tags/ipp-20120216/ippToPsps/config/settings.xml (revision 33342) +++ /tags/ipp-20120216/ippToPsps/config/settings.xml (revision 33342) @@ -0,0 +1,42 @@ + + + + + + + + /data/ipp005.0/rhenders + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + Index: /tags/ipp-20120216/ippToPsps/jython/batch.py =================================================================== --- /tags/ipp-20120216/ippToPsps/jython/batch.py (revision 33341) +++ /tags/ipp-20120216/ippToPsps/jython/batch.py (revision 33342) @@ -377,5 +377,10 @@ self.logger.infoPair("Importing tables with filter", filter) - tables = stilts.treads(self.fits.getPath()) + + try: + tables = stilts.treads(self.fits.getPath()) + except: + self.logger.errorPair("STILTS could not import from", self.fits.getPath()) + return False count = 0 Index: /tags/ipp-20120216/ippToPsps/jython/dvo.py =================================================================== --- /tags/ipp-20120216/ippToPsps/jython/dvo.py (revision 33341) +++ /tags/ipp-20120216/ippToPsps/jython/dvo.py (revision 33342) @@ -39,5 +39,4 @@ self.logger = logger self.config = config - self.logger.infoSeparator() # create database object @@ -56,5 +55,5 @@ if not self.correctDvo: print "*******************************************************************************" - response = raw_input("**** Wrong DVO in use. Do you want to reset and use '" + self.config.dvoLocation + "' instead (y/n)? ") + response = raw_input("**** Wrong or no DVO in use. Do you want to reset and use '" + self.config.dvoLocation + "' (y/n)? ") if response == "y": response = raw_input("**** Are you ABSOLUTELY sure you want to do this? (y/n)? ") @@ -180,7 +179,5 @@ halfSide = side/2.0 - print "%f %f %f" % (ra, dec, side) self.setSkyArea(ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide) - ''' @@ -198,5 +195,5 @@ # update the two meta tables, Images and SkyTable - self.loadImages() + #self.loadImages() dont need a new Images table at this point self.loadSkyTable() @@ -346,4 +343,7 @@ self.logger.infoPair("Nothing to ingest or purge", "finished ingest") return True + + # make sure we have an up-to-date Images table + self.loadImages() # purge stuff Index: /tags/ipp-20120216/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /tags/ipp-20120216/ippToPsps/jython/ipptopspsdb.py (revision 33341) +++ /tags/ipp-20120216/ippToPsps/jython/ipptopspsdb.py (revision 33342) @@ -717,4 +717,58 @@ ''' + Remove client + ''' + def removeClient(self, host, pid): + self.execute("DELETE FROM clients WHERE host = '" + host + "' AND pid = " + str(pid)) + + ''' + Insert client + ''' + def insertClient(self, host, pid): + self.execute("INSERT INTO clients (timestamp, host, pid) VALUES (now(), '" + host + "', " + str(pid) + ")") + + ''' + Update clients, or inserts it for the first time + ''' + def updateClient(self, host, pid): + + try: + self.insertClient(host, pid) + except: + self.execute("UPDATE clients SET timestamp = now() WHERE host = '" + host + "' AND pid = " + str(pid)) + + ''' + Is this client paused? + ''' + def isPaused(self, host, pid): + return self.getClientsColumnBoolean("paused", host, pid) + + ''' + Is this client killed? + ''' + def isKilled(self, host, pid): + return self.getClientsColumnBoolean("killed", host, pid) + + ''' + Returns boolean value for this column in clients table + ''' + def getClientsColumnBoolean(self, col, host, pid): + + sql = "SELECT " + col + " \ + FROM clients \ + WHERE host = '" + host + "' AND pid = " + str(pid) + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) == 1: return True + else: return False + except: + self.logger.exception("Unable to check whether " + col + " is true or false in clients table") + + return False + + + ''' Destructor ''' Index: /tags/ipp-20120216/ippToPsps/jython/load.py =================================================================== --- /tags/ipp-20120216/ippToPsps/jython/load.py (revision 33341) +++ /tags/ipp-20120216/ippToPsps/jython/load.py (revision 33342) @@ -5,8 +5,8 @@ # +import signal import time import sys import os -import socket import logging.config @@ -115,4 +115,7 @@ batch.run() + # check that we should continue + checkClientStatus() + ippToPspsDb.lockBatchTable() @@ -123,4 +126,39 @@ ippToPspsDb.unlockTables() + +''' +Handler for Ctrl-C signal +''' +def signal_handler(signal, frame): + + exitProgram("Ctrl-C") + +''' +Cleanly exits program, prints to log the way it exited +''' +def exitProgram(exitType): + + logger.infoPair("Program exited", exitType) + ippToPspsDb.removeClient(logger.host, logger.pid) + sys.exit(0) + + +''' +Checks if we should pause or kill loading, otherwise updates the clients table with current time +''' +def checkClientStatus(): + + ippToPspsDb.updateClient(logger.host, logger.pid) + + if ippToPspsDb.isKilled(logger.host, logger.pid): exitProgram("killed") + + PAUSEPERIOD = 60 + while ippToPspsDb.isPaused(logger.host, logger.pid): + + ippToPspsDb.updateClient(logger.host, logger.pid) + if ippToPspsDb.isKilled(logger.host, logger.pid): exitProgram("killed while paused") + logger.infoPair("Program paused for", "%d secs" % PAUSEPERIOD) + time.sleep(PAUSEPERIOD) + ''' @@ -139,6 +177,5 @@ # open config file config = Config(CONFIGPATH) -#logger = config.getLogger(sys.argv[0], 0, 1) -logger = config.getLogger(sys.argv[0]) +logger = config.getLogger(sys.argv[0], 1, 1) # create various objects @@ -148,9 +185,13 @@ datastore = Datastore(logger, config, ippToPspsDb) +checkClientStatus() + +# catch Ctrl-C signal +signal.signal(signal.SIGINT, signal_handler) + # check we connected ok if not gpc1Db.everythingOK: sys.exit(1) if not ippToPspsDb.everythingOK: sys.exit(1) -# get values from the configutaion file POLLPERIOD = 600 @@ -178,4 +219,5 @@ sys.exit(1) + ''' Main processing loop: @@ -206,4 +248,6 @@ while dec <= config.maxDec: + + checkClientStatus() # for each batch type @@ -228,5 +272,5 @@ # do we pre-ingest stuff from DVO? - if smfsPerGB > 0: + if smfsPerGB > 40: if not dvo.sync(): logger.errorPair("Could not sync DVO with MySQL", "skipping") @@ -238,5 +282,5 @@ logger.infoBool("Using pre-ingested DVO data?", useFullTables) processTheseItems(batchType, ids, useFullTables) - + # in the test mode, we quit after submitting one batch if config.test: break @@ -253,3 +297,4 @@ time.sleep(POLLPERIOD) - +exitProgram("end of program") + Index: /tags/ipp-20120216/ippToPsps/jython/metrics.py =================================================================== --- /tags/ipp-20120216/ippToPsps/jython/metrics.py (revision 33341) +++ /tags/ipp-20120216/ippToPsps/jython/metrics.py (revision 33342) @@ -55,5 +55,6 @@ # get a master list of IDs in DVO for this batch type - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(config.dvoLabel, batchType, config.minRa, config.maxRa, config.minDec, config.maxDec) + #allIDs = gpc1Db.getIDsInThisDVODbForThisStage(config.dvoLabel, batchType, config.minRa, config.maxRa, config.minDec, config.maxDec) + allIDs = gpc1Db.getIDsInThisDVODbForThisStage(config.dvoLabel, batchType) prevList = allIDs numEverything = len(allIDs) @@ -185,4 +186,4 @@ time.sleep(SECONDS) else: break + config.refresh() - Index: /tags/ipp-20120216/ippToPsps/jython/pslogger.py =================================================================== --- /tags/ipp-20120216/ippToPsps/jython/pslogger.py (revision 33341) +++ /tags/ipp-20120216/ippToPsps/jython/pslogger.py (revision 33342) @@ -3,4 +3,5 @@ import os import sys +import socket class PSLogger(logging.getLoggerClass()): @@ -17,11 +18,11 @@ if sendToFile: - HOST = socket.gethostname() - PID = os.getpid() + self.host = socket.gethostname() + self.pid = os.getpid() # generate path PATH = basePath + "/log" if not os.path.exists(PATH): os.makedirs(PATH) - FULLPATH = PATH + "/" + name + "_" + dvoLabel + "_" + HOST + "_" + str(PID) + ".log" + FULLPATH = PATH + "/" + name + "_" + dvoLabel + "_" + self.host + "_" + str(self.pid) + ".log" # opens file to be appended Index: /tags/ipp-20120216/ippToPsps/jython/scratchdb.py =================================================================== --- /tags/ipp-20120216/ippToPsps/jython/scratchdb.py (revision 33341) +++ /tags/ipp-20120216/ippToPsps/jython/scratchdb.py (revision 33342) @@ -330,5 +330,5 @@ except: self.logger.errorPair("No DVO files ingested for this path", path) - return True + return False ''' Index: /tags/ipp-20120216/ippTools/src/camtool.c =================================================================== --- /tags/ipp-20120216/ippTools/src/camtool.c (revision 33341) +++ /tags/ipp-20120216/ippTools/src/camtool.c (revision 33342) @@ -112,4 +112,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); if (!psListLength(where->list) && Index: /tags/ipp-20120216/ippTools/src/chiptool.c =================================================================== --- /tags/ipp-20120216/ippTools/src/chiptool.c (revision 33341) +++ /tags/ipp-20120216/ippTools/src/chiptool.c (revision 33342) @@ -198,4 +198,11 @@ GET_VALUE (psS64, S64, exp_id, "exp_id"); GET_VALUE (psString, Str, raw_workdir, "workdir"); +#ifdef notyet + GET_VALUE (psString, Str, old_workdir, "old_workdir"); + GET_VALUE (psString, Str, old_data_group, "old_data_group"); +#else + const char *old_workdir = NULL; + const char *old_data_group = NULL; +#endif GET_VALUE (psString, Str, raw_label, "label"); GET_VALUE (psString, Str, raw_reduction, "reduction"); @@ -213,7 +220,7 @@ if (!pxchipQueueByExpTag(config, exp_id, - workdir ? workdir : raw_workdir, + workdir ? workdir : (old_workdir ? old_workdir : raw_workdir), label ? label : raw_label, - data_group ? data_group : (label ? label : raw_label), + data_group ? data_group : (old_data_group ? old_data_group : (label ? label : raw_label)), dist_group, reduction ? reduction : raw_reduction, @@ -345,4 +352,5 @@ pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); pxAddLabelSearchArgs (config, where, "-data_group", "chipOld.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "chipOld.state", "=="); // psListLength(where->list) is at least 1 because exp_type defaults to "object" @@ -354,5 +362,5 @@ } - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); @@ -468,4 +476,5 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); psString query = pxDataGet("chiptool_pendingimfile.sql"); @@ -2024,7 +2033,7 @@ return false; } - psFree(setHook); psFree(query); + // if we are updating a whole chipRun set lingering chips with poor quality to full data_state if (!class_id) { Index: /tags/ipp-20120216/ippTools/src/chiptoolConfig.c =================================================================== --- /tags/ipp-20120216/ippTools/src/chiptoolConfig.c (revision 33341) +++ /tags/ipp-20120216/ippTools/src/chiptoolConfig.c (revision 33342) @@ -72,6 +72,7 @@ psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); - - psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-state", 0, "search by chipRun.state", NULL); + + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); @@ -111,4 +112,5 @@ psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", 0); pxchipSetSearchArgs(pendingimfileArgs); psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: /tags/ipp-20120216/ippTools/src/laptool.c =================================================================== --- /tags/ipp-20120216/ippTools/src/laptool.c (revision 33341) +++ /tags/ipp-20120216/ippTools/src/laptool.c (revision 33342) @@ -390,4 +390,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); psMetadata *where = psMetadataAlloc(); @@ -411,4 +412,9 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); } Index: /tags/ipp-20120216/ippTools/src/laptoolConfig.c =================================================================== --- /tags/ipp-20120216/ippTools/src/laptoolConfig.c (revision 33341) +++ /tags/ipp-20120216/ippTools/src/laptoolConfig.c (revision 33342) @@ -86,4 +86,5 @@ ADD_OPT(Str, listrunArgs, "-fault", "search by LAP run fault", NULL); ADD_OPT(Bool,listrunArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, listrunArgs, "-limit", "limit result set to N items", 0); // -updaterun Index: /tags/ipp-20120216/ippTools/src/pstamptool.c =================================================================== --- /tags/ipp-20120216/ippTools/src/pstamptool.c (revision 33341) +++ /tags/ipp-20120216/ippTools/src/pstamptool.c (revision 33342) @@ -780,4 +780,5 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + PXOPT_COPY_STR(config->args, where, "-jobType", "jobType", "=="); PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); Index: /tags/ipp-20120216/ippTools/src/pstamptoolConfig.c =================================================================== --- /tags/ipp-20120216/ippTools/src/pstamptoolConfig.c (revision 33341) +++ /tags/ipp-20120216/ippTools/src/pstamptoolConfig.c (revision 33342) @@ -154,4 +154,5 @@ psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-dep_id", 0, "select by dependent ID", 0); psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); + psMetadataAddStr(listjobArgs, PS_LIST_TAIL, "-jobType", 0, "select by jobType", 0); psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: /tags/ipp-20120216/ippTools/src/pxchip.c =================================================================== --- /tags/ipp-20120216/ippTools/src/pxchip.c (revision 33341) +++ /tags/ipp-20120216/ippTools/src/pxchip.c (revision 33342) @@ -71,5 +71,5 @@ psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); - psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); Index: /tags/ipp-20120216/ippconfig/gpc1/psastro.config =================================================================== --- /tags/ipp-20120216/ippconfig/gpc1/psastro.config (revision 33341) +++ /tags/ipp-20120216/ippconfig/gpc1/psastro.config (revision 33342) @@ -344,4 +344,6 @@ PSASTRO.MATCH.RADIUS.N7 F32 5 PSASTRO.MATCH.FIT.NITER S32 5 + + PSASTRO.CATDIR STR STS.PP5.REFCAT END Index: /tags/ipp-20120216/pstamp/scripts/Makefile.am =================================================================== --- /tags/ipp-20120216/pstamp/scripts/Makefile.am (revision 33341) +++ /tags/ipp-20120216/pstamp/scripts/Makefile.am (revision 33342) @@ -29,4 +29,5 @@ detect_query_create \ detect_response_create \ + dquery_job_run.pl \ dquery_finish.pl \ dqueryparse.pl \ Index: /tags/ipp-20120216/pstamp/scripts/dquery_finish.pl =================================================================== --- /tags/ipp-20120216/pstamp/scripts/dquery_finish.pl (revision 33341) +++ /tags/ipp-20120216/pstamp/scripts/dquery_finish.pl (revision 33342) @@ -143,5 +143,5 @@ if (-s $reg_list) { - my $command = "echo Skipping $dsreg --add $req_name --product $product --list $reg_list"; + my $command = "$dsreg --add $req_name --product $product --list $reg_list"; $command .= " --copy --datapath $outdir"; $command .= " --type MOPS_DETECTABILITY_RESPONSE"; Index: /tags/ipp-20120216/pstamp/scripts/dqueryparse.pl =================================================================== --- /tags/ipp-20120216/pstamp/scripts/dqueryparse.pl (revision 33341) +++ /tags/ipp-20120216/pstamp/scripts/dqueryparse.pl (revision 33342) @@ -100,10 +100,15 @@ +# +# query is a hash which uses $fpa_id as the key +# The values are hash references which have the various parameter name as the key +# The values of the parameter hashes are arrays which contain the values for individual rows +# in the detectabilty query request my %query = (); -my %image_list_hash = (); { # # Parse input request file using detect_query_read + # result is a simple text file # my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file"; @@ -139,5 +144,5 @@ } # - # Identify target images. This should properly collate targets on a single imfile. + # Identify target components. This should properly collate targets on a single imfile. # my $query_id; @@ -223,5 +228,7 @@ $option_mask |= $PSTAMP_SELECT_VARIANCE; $option_mask |= $PSTAMP_SELECT_PSF; - my $need_magic = 1; + + # magic is dead + my $need_magic = 0; if ($stage eq 'stack') { $need_magic = 0; @@ -245,15 +252,5 @@ foreach my $this_image_ref (@{ $pstamp_images_ref }) { - if (0) { - foreach my $key (sort (keys %{ $this_image_ref } )) { - my $value = ${ $this_image_ref }{$key}; - if ($key eq 'row_index') { - $value = join ' ', @{ $this_image_ref->{$key} }; - } - print "$this_image_ref $key $value\n"; - } - } - - + # loop over the rows that this component matched foreach my $valid_index (@{ $this_image_ref->{row_index} }) { $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; @@ -336,18 +333,4 @@ foreach my $fpa_id (keys %query) { for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) { -# print WISDOM "$fpa_id\t"; -# foreach my $key (keys %{ $query{$fpa_id} }) { -# print WISDOM "$key $query{$fpa_id}{$key}[$i]\t"; -# } -# print WISDOM "\n"; -# my $data_state = $query{$fpa_id}{DATA_STATE}[$i]; -# if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { -# if ($data_state ne 'full') { - -# @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = -# ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], -# $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); -# } -# } if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { push @{ $rows_for_component{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; @@ -358,5 +341,9 @@ } -my $job_num = 0; +# +# Finally build the parameter lists and queue jobs for each component +# + +my $job_num = 1; foreach my $fpa_id (keys %rows_for_component) { foreach my $component (keys % {$rows_for_component{$fpa_id} } ) { @@ -368,13 +355,18 @@ my $stage = $query{$fpa_id}{STAGE}[$i]; my $stage_id = $query{$fpa_id}{STAGE_ID}[$i]; - my $component = $query{$fpa_id}{STAGE_ID}[$i]; + my $component = $query{$fpa_id}{COMPONENT_ID}[$i]; print "Need to UPDATE $stage $stage_id $component from $data_state\n"; + #### my ($req_id, $outdir, $label, $state, $stage, $stage_id, $component, $need_magic, $imagedb) = @_; $dep_id = queue_update_run($req_id, $outdir, $label, $data_state, $stage, $stage_id, $component, 0, $imagedb); } my $outputBase = "$outdir/$job_num" . "_"; + + # params_file is an mdc description containing the parameters for the job my $params_file = $outputBase . "params.mdc"; open PARAMS, ">$params_file" or my_die("failed to create $params_file", $PS_EXIT_UNKNOWN_ERROR); + + # first the job params print PARAMS "dqueryJobParams METADATA\n"; foreach my $key (keys %{ $query{$fpa_id} } ) { @@ -383,8 +375,10 @@ } print PARAMS "END\n\n"; + + # then one structure with the coordinates for each row print PARAMS "dqueryCoord MULTI\n"; + my @coordkeys = qw(ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG); foreach my $row ( @{ $rows_for_component{$fpa_id}{$component} }) { - my @coordkeys = qw(ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG); print PARAMS "\ndqueryCoord METADATA\n"; foreach my $key (@coordkeys) { Index: /tags/ipp-20120216/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /tags/ipp-20120216/pstamp/scripts/pstamp_job_run.pl (revision 33341) +++ /tags/ipp-20120216/pstamp/scripts/pstamp_job_run.pl (revision 33342) @@ -79,5 +79,5 @@ my $ppstamp = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1); my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1); -my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1); +my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1); my $streaksreplace = can_run('streaksreplace') or (warn "Can't find streaksreplace" and $missing_tools = 1); my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); @@ -303,18 +303,15 @@ } } elsif ($jobType eq "detect_query") { - # detect_query jobs are basically holders to note that we need a file updated before we can continue. - # Load the argument list that dqueryparse should have created the first time it ran, so we know how to - # run it again the same way. - my $outdir = dirname($outputBase); - my $argslist = "$outdir/parse.args"; - open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR); - my $argString = ; - close ARGSLIST; - chomp $argString; + # my $outdir = dirname($outputBase); + # my $argslist = "$outdir/parse.args"; + # open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR); + # my $argString = ; + # close ARGSLIST; + # chomp $argString; # XXX: should we do any other sanity checking? - my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString; - - my $command = "$dqueryparse --job_id $job_id $argString"; + # my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString; + + my $command = "$dquery_job_run --job_id $job_id --output_base $outputBase"; $command .= " --dbname $dbname" if $dbname; $command .= " --dbserver $dbserver" if $dbserver; @@ -329,5 +326,5 @@ } else { $jobStatus = $error_code >> 8; - my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus); + my_die("dquery_job_run.pl failed with error code: $jobStatus", $job_id, $jobStatus); } } elsif ($jobType eq "child") { Index: /tags/ipp-20120216/tools/czarcal =================================================================== --- /tags/ipp-20120216/tools/czarcal (revision 33341) +++ /tags/ipp-20120216/tools/czarcal (revision 33342) @@ -29,5 +29,4 @@ heather bill -roy );