Index: /branches/eam_branches/ipp-20111122/dbconfig/add.md =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/add.md (revision 32851) +++ /branches/eam_branches/ipp-20111122/dbconfig/add.md (revision 32852) @@ -3,5 +3,6 @@ add_id S64 0 # Primary Key AUTO_INCREMENT stage STR 64 # what the stage is (warp, cam, diff, - stage_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) + stage_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) + stage_extra1 S32 0 state STR 64 # Key workdir STR 255 Index: /branches/eam_branches/ipp-20111122/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/changes.txt (revision 32851) +++ /branches/eam_branches/ipp-20111122/dbconfig/changes.txt (revision 32852) @@ -2177,2 +2177,9 @@ UPDATE dbversion set schema_version = '1.1.71', updated= CURRENT_TIMESTAMP(); + + +-- this is to stuff extra stage related information. +-- example - multi filter staticsky stage - this allows selection of +-- each filter of cmf staticsky. + +alter table addRun add column stage_extra1 int after stage_id; Index: /branches/eam_branches/ipp-20111122/dbconfig/pstamp.md =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/pstamp.md (revision 32851) +++ /branches/eam_branches/ipp-20111122/dbconfig/pstamp.md (revision 32852) @@ -67,2 +67,8 @@ num S64 0 # Primary Key AUTO_INCREMENT END + +pstampFile METADATA + file_id S64 0 # Primary Key AUTO_INCREMENT + job_id S64 0 + path STR 255 +end Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py (revision 32852) @@ -286,5 +286,5 @@ Publishes this batch to the datastore ''' - def tarZipAndpublishToDatastore(self): + def tarAndZip(self): # set up filenams and paths @@ -295,12 +295,25 @@ # tar directory - p = Popen("tar -cvf " + tarPath + "\ - -C " + self.subDir + " \ - " + self.batchName, shell=True, stdout=PIPE) + cmd = "tar -cvf " + tarPath + " -C " + self.subDir + " " + self.batchName + self.logger.infoPair("Creating tar archive", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("tar command", "failed") + return False + # zip tar archive - p = Popen("gzip -c " + tarPath + " > " + tarballPath, shell=True, stdout=PIPE) + cmd = "gzip -c " + tarPath + " > " + tarballPath + self.logger.infoPair("Compressing tar archive", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + + if p.returncode != 0: + self.logger.errorPair("gzip command", "failed") + return False + + # only now can we report that the batch has successfully processed + self.ippToPspsDb.updateProcessed(self.batchID, 1) # delete tar file and original directory @@ -308,6 +321,5 @@ shutil.rmtree(self.localOutPath) - # now publish to the datastore - Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile) + return True ''' @@ -448,5 +460,4 @@ try: stilts.twrites(_tables, self.outputFitsPath, fmt='fits') - self.ippToPspsDb.updateProcessed(self.batchID, 1) except: self.logger.exception("Could not write to FITS") @@ -494,5 +505,4 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() - # out = p.stdout.read() rowCount = self.scratchDb.getRowCount("dvoDetection") @@ -507,5 +517,5 @@ ''' Creates and publishes a batch - TODO all methdds call below should throw exceptions on failure + TODO all methods call below should throw exceptions on failure ''' def run(self): @@ -530,5 +540,11 @@ else: self.writeBatchManifest() - if int(self.doc.find("options/publishToDatastore").text): self.tarZipAndpublishToDatastore() + if int(self.doc.find("options/publishToDatastore").text): + # tar and zip ready for publication to datastore + if self.tarAndZip(): + # now publish to the datastore + tarballFile = Batch.getTarballFile(self.batchID) + Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile) + if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False) Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newsas.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newsas.xml (revision 32852) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newsas.xml (revision 32852) @@ -0,0 +1,90 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + SAS.20111125 + /data/ipp005.0/gpc1/catdirs/SAS.20111125 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-12-01 + 3PI + SA3 + 1 + 1 + 0 + 0 + 1 + 0 + + + + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi.xml (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi.xml (revision 32852) @@ -48,5 +48,5 @@ ThreePi.V3 /data/ipp005.0/gpc1/catdirs/ThreePi.V3 - 135 + 105 270 -100 Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/czardb.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/czardb.py (revision 32852) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/czardb.py (revision 32852) @@ -0,0 +1,50 @@ +#!/usr/bin/env jython + +import re +import sys +import os +import logging + +from mysql import MySql +from java.sql import * + +''' +Class for CzarDb database connectivity +''' +class CzarDb(MySql): + + ''' + Constructor + ''' + def __init__(self, logger, doc): + super(CzarDb, self).__init__(logger, doc, "czardatabase") + + + ''' + Updates a stage table with pending, processed, faults and label + ''' + def insertStats(self, stage, label, batchType, pending, processed, faults): + + sql = "INSERT INTO " + stage + " \ + (label, pending, processed, faults) \ + VALUES ('" + label + "_" + batchType + "', \ + " + str(pending) + ", \ + " + str(processed) + ", \ + " + str(faults) + ")" + + try: + self.execute(sql) + except: + pass + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("CzarDb Desstructor") + self.stmt.close() + self.con.close() + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastore.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastore.py (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastore.py (revision 32852) @@ -48,5 +48,5 @@ # build dsreg command command = "dsreg --add " + name + "\ - --copy --datapath " + path + "\ + --link --datapath " + path + "\ --type " + self.type + "\ --product " + self.product + "\ Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastoreDupeChecker.sh =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastoreDupeChecker.sh (revision 32852) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastoreDupeChecker.sh (revision 32852) @@ -0,0 +1,11 @@ +rm index.txt +wget http://datastore.ipp.ifa.hawaii.edu/PSPS_JHU/index.txt + +echo "Dupes:" +echo "" + +more index.txt | awk '{print $1}' | uniq -d +echo "" +echo "Done" + +#wget http://datastore.ipp.ifa.hawaii.edu/PSPS_JHU/B00263471/index.txt Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py (revision 32852) @@ -30,32 +30,4 @@ self.logger.debug("Gpc1Db destructor") - - ''' - TODO - ''' - def getIDsInThisDVODbForThisStageFudge(self): - - sql = "SELECT staticskyRun.sky_id \ - FROM staticskyInput, staticskyRun, stackRun, staticskyResult \ - WHERE staticskyRun.sky_id = staticskyInput.sky_id \ - AND staticskyInput.stack_id = stackRun.stack_id \ - AND staticskyInput.sky_id = staticskyResult.sky_id \ - AND staticskyRun.label like 'MD04.staticsky' \ - AND stackRun.filter like 'i%'" - - try: - rs = self.executeQuery(sql) - except: - self.logger.exception("Can't query for ids in DVO") - - ids = [] - while (rs.next()): - ids.append(rs.getInt(1)) - - rs.close() - - self.logger.debug("Found %d items in DVO database '" % (len(ids))) - - return ids ''' Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopspsdb.py (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopspsdb.py (revision 32852) @@ -103,4 +103,10 @@ ''' Returns a list of processed batch IDs that are not loaded to the datastore + NB we have a date constraint here because, before this time, it is possible that + a test batch would be labelled as processed but not loaded to datastore, and we would NOT + want to load a test batch to PSPS + + We also check that these batches are more than 4 HOURs old, so that we don't attempt to + publish something simulataneously with the client that it producing it ''' def getProcessedButFailedDatastoreBatchIDs(self, epoch, dvoGpc1Label, batchType): @@ -112,5 +118,7 @@ AND dvo_db = '" + dvoGpc1Label + "' \ AND processed = 1\ - AND loaded_to_datastore = -1" + AND loaded_to_datastore != 1 \ + AND timestamp > '2011-10-27' \ + AND timestamp < now() - INTERVAL 4 HOUR" ids = [] @@ -466,5 +474,5 @@ AND batch_type = '" + batchType + "' \ AND dvo_db = '" + dvoGpc1Label + "' \ - AND timestamp BETWEEN now() - INTERVAL 2 HOUR AND now()" + AND timestamp BETWEEN now() - INTERVAL 4 HOUR AND now()" try: @@ -499,5 +507,5 @@ ''' - Have we already processed and published this batch? + Have we already processed this stage_id? ''' def alreadyProcessed(self, batchType, stage_id, epoch, dvoGpc1Label): @@ -509,6 +517,5 @@ AND timestamp > '" + epoch + "' \ AND dvo_db = '" + dvoGpc1Label + "' \ - AND processed = 1 \ - AND loaded_to_datastore != 0" + AND processed = 1" try: @@ -655,4 +662,18 @@ self.execute(sql) + ''' + Resets a batch ready for re-loading to the datastore + ''' + def resetBatch(self, batchID): + + sql = "UPDATE batch SET \ + loaded_to_datastore = 0, \ + loaded_to_ODM = 0, \ + deleted_datastore = 0, \ + deleted_dxlayer = 0, \ + comment = 'This batch was reset, ready for re-load to datastore' \ + WHERE batch_id = " + str(batchID) + + self.execute(sql) ''' Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/metrics.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/metrics.py (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/metrics.py (revision 32852) @@ -1,3 +1,4 @@ import math +import time import logging import smtplib @@ -10,4 +11,5 @@ from pslogger import PSLogger from ipptopspsdb import IppToPspsDb +from czardb import CzarDb from gpc1db import Gpc1Db @@ -76,6 +78,6 @@ numPending = len(pending) - #if stage == 'merge_worthy' and batchType == "P2": - # for n in pending: print n + #if stage == 'loaded_to_datastore' and batchType == "P2": + # for n in fail: print n str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m " % (getIntAsString(numPending), getIntAsString(numSuccess), getIntAsString(numFail)) @@ -84,4 +86,6 @@ prevList = success print >> DATFILE, stage, numSuccess, numFail, numPending + + czarDb.insertStats(stage, DVOLABEL, batchType, numPending, numSuccess, numFail) if stage == 'processed': numPendingProcessed = numPending @@ -124,8 +128,14 @@ if len(sys.argv) > 1: CONFIG = sys.argv[1] else: - print "** Usage: " + sys.argv[0] + " " + print "** Usage: " + sys.argv[0] + " [hours]" sys.exit(1) -now = datetime.datetime.now() +if len(sys.argv) > 2: + HOURS = float(sys.argv[2]) + SECONDS = HOURS * 60.0 * 60.0 +else: + SECONDS = None + + # open config file @@ -136,56 +146,70 @@ logger.setup(configDoc, "metrics") +# create database objects ippToPspsDb = IppToPspsDb(logger, configDoc) +czarDb = CzarDb(logger, configDoc) gpc1Db = Gpc1Db(logger, configDoc) - -MINRA = None -MAXRA = None -MINDEC = None -MAXDEC = None DVOLABEL = configDoc.find("dvo/gpc1Label").text EPOCH = configDoc.find("options/epoch").text -logger.infoTitle("ippToPsps loading summary") -logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S")) -logger.infoPair("Loading epoch", EPOCH) -logger.infoPair("DVO label", DVOLABEL) - -try: - MINRA = float(configDoc.find("dvo/minRA").text) - MAXRA = float(configDoc.find("dvo/maxRA").text) - MINDEC = float(configDoc.find("dvo/minDec").text) - MAXDEC = float(configDoc.find("dvo/maxDec").text) - logger.infoPair("Coordinates", "%.1f to %.1f RA, %.1f to %.1f Dec" % (MINRA, MAXRA, MINDEC, MAXDEC)) -except: - pass +while True: + + logger.infoTitle("ippToPsps loading summary") + now = datetime.datetime.now() + logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S")) + logger.infoPair("Loading epoch", EPOCH) + logger.infoPair("DVO label", DVOLABEL) + + # get equatorial coord constraints, if any + MINRA = None + MAXRA = None + MINDEC = None + MAXDEC = None + + try: + MINRA = float(configDoc.find("dvo/minRA").text) + MAXRA = float(configDoc.find("dvo/maxRA").text) + MINDEC = float(configDoc.find("dvo/minDec").text) + MAXDEC = float(configDoc.find("dvo/maxDec").text) + logger.infoPair("Coordinates", "%.1f to %.1f RA, %.1f to %.1f Dec" % (MINRA, MAXRA, MINDEC, MAXDEC)) + except: + pass -logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") -logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |") -logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") -rateP2 = printStats("P2") -rateST = printStats("ST") -logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") - -logger.info("") - -stages = ippToPspsDb.getStages() -colCount = len(stages) -writetableSeparator(colCount) -sys.stdout.write("|Type| DVO ") -for stage in stages: sys.stdout.write("|%19s" % stage) -sys.stdout.write("|\n") -sys.stdout.write("| | ") -for stage in stages: sys.stdout.write("| Pend Succ Fail ") -sys.stdout.write("|\n") -writetableSeparator(colCount) -pendP2Processed = printTableRow(stages, "P2") -pendSTProcessed = printTableRow(stages, "ST") -writetableSeparator(colCount) - -logger.info("") -try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) -except: pass -try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) -except: pass + logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |") + logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + rateP2 = printStats("P2") + rateST = printStats("ST") + logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + + logger.info("") + + stages = ippToPspsDb.getStages() + colCount = len(stages) + writetableSeparator(colCount) + sys.stdout.write("|Type| DVO ") + + for stage in stages: sys.stdout.write("|%19s" % stage) + sys.stdout.write("|\n") + sys.stdout.write("| | ") + for stage in stages: sys.stdout.write("| Pend Succ Fail ") + sys.stdout.write("|\n") + writetableSeparator(colCount) + pendP2Processed = printTableRow(stages, "P2") + pendSTProcessed = printTableRow(stages, "ST") + writetableSeparator(colCount) + + logger.info("") + try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) + except: pass + try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) + except: pass + + if SECONDS: + logger.infoPair("Waiting for", "%f hours" % HOURS) + time.sleep(SECONDS) + else: break + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/odm.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/odm.py (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/odm.py (revision 32852) @@ -67,4 +67,6 @@ STATE = doc.find("{%s}OdmBatchState/{%s}BatchState" % (NAMESPACE, NAMESPACE)).text MESSAGE = doc.find("{%s}OdmBatchState/{%s}Message" % (NAMESPACE, NAMESPACE)).text + results['MESSAGE'] = MESSAGE + results['DETAILS'] = "None" # we assume that batches 'OnHold' are failed as it is very rare for such Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/republishbatch.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/republishbatch.py (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/republishbatch.py (revision 32852) @@ -11,5 +11,7 @@ ''' +A program that republishes a bath to the datastore. +It first 'resets' the batch in the ippToPsps database, i.e. pretends it has never been published, then re-publishes it to the datastore ''' if len(sys.argv) > 2: @@ -50,4 +52,6 @@ logger.infoPair("subdir", subDir) logger.infoPair("tarballfile", tarballFile) +logger.infoPair("Reseting batch in database", "%d" % BATCHID) +ippToPspsDb.resetBatch(BATCHID) logger.infoPair("publishing to", DATASTOREPRODUCT) Batch.publishToDatastore(datastore, BATCHID, batchName, subDir, tarballFile) Index: /branches/eam_branches/ipp-20111122/ippToPsps/xmlschema/BatchManifest.xsd =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/xmlschema/BatchManifest.xsd (revision 32851) +++ /branches/eam_branches/ipp-20111122/ippToPsps/xmlschema/BatchManifest.xsd (revision 32852) @@ -51,5 +51,10 @@ - + + + + + + Index: /branches/eam_branches/ipp-20111122/ppSub/src/ppSubFlagNeighbors.c =================================================================== --- /branches/eam_branches/ipp-20111122/ppSub/src/ppSubFlagNeighbors.c (revision 32851) +++ /branches/eam_branches/ipp-20111122/ppSub/src/ppSubFlagNeighbors.c (revision 32852) @@ -18,8 +18,11 @@ #include "ppSub.h" +# define MIN_ERROR 0.02 /* padding for self-match flux test */ +# define MAX_OFFSET 5.0 /* delta mag greater than this is not a self match */ + // match the diff sources to the detections from the positive or negative input images // matchRef : false if this is the standard diff image, true if this is the inverse (in-ref vs ref-in) // this is needed to choose which value is positive and which is negative relative to the difference -bool ppSubFlagNeighbors(pmConfig *config, pmFPAview *view, psArray *sources, bool matchRef) { +bool ppSubFlagNeighbors(pmConfig *config, pmFPAview *view, psArray *sources, bool subInverse) { bool status; @@ -106,10 +109,24 @@ psAssert ((sourceM1->imageID == 1) || (sourceM1->imageID == 2), "error in pmPhotObj construction (second entry is not from 1 or 2"); + // which image gives the match, the positive or negative one? + bool positive = false; + if (subInverse) { + positive = (sourceM1->imageID == 2); + } else { + positive = (sourceM1->imageID == 1); + } + + // check if source and sourceM1 are likely to be the same flux (ie, isolated detection in one image) + if (positive && isfinite(sourceM1->psfMag) && isfinite(source->psfMag)) { + float dMag = sourceM1->psfMag - source->psfMag; + float nSig = fabs(dMag) / hypot(MIN_ERROR, source->psfMagErr); + if (nSig < MAX_OFFSET) { + source->mode2 |= PM_SOURCE_MODE2_DIFF_SELF_MATCH; + } + } + // only one match. set a flag? source->mode2 |= PM_SOURCE_MODE2_DIFF_WITH_SINGLE; if (source->diffStats) { - // which is match need an xor here... - bool positive = !matchRef && (sourceM1->imageID == 1); - positive |= matchRef && (sourceM1->imageID == 2); float SN1 = isfinite(sourceM1->psfMagErr) ? 1.0 / sourceM1->psfMagErr : NAN; if (positive) { @@ -133,9 +150,26 @@ psAssert (sourceM1->imageID != sourceM2->imageID, "error in pmPhotObj construction (case 3.3)"); + // we don't know which matched source (sourceM1 or sourceM2) is the positive detection, + // and the answer depends on both the ID and the subtraction order (in-ref vs ref-in) + // positive is true if sourceM1 is the positive detection + bool positive = false; + if (subInverse) { + positive = (sourceM1->imageID == 2); + } else { + positive = (sourceM1->imageID == 1); + } + + // check for self-detection + float magPSF1 = positive ? sourceM1->psfMag : sourceM2->psfMag; + if (isfinite(magPSF1) && isfinite(source->psfMag)) { + float dMag = magPSF1 - source->psfMag; + float nSig = fabs(dMag) / hypot(MIN_ERROR, source->psfMagErr); + if (nSig < MAX_OFFSET) { + source->mode2 |= PM_SOURCE_MODE2_DIFF_SELF_MATCH; + } + } + source->mode2 |= PM_SOURCE_MODE2_DIFF_WITH_DOUBLE; if (source->diffStats) { - // which is match need an xor here... - bool positive = !matchRef && (sourceM1->imageID == 1); - positive |= matchRef && (sourceM1->imageID == 2); float SN1 = isfinite(sourceM1->psfMagErr) ? 1.0 / sourceM1->psfMagErr : NAN; float SN2 = isfinite(sourceM2->psfMagErr) ? 1.0 / sourceM2->psfMagErr : NAN; @@ -187,4 +221,6 @@ pmPhotObj *obj = objects->data[j]; + // fprintf (stderr, "%f, %f vs %f, %f\n", src->peak->xf, src->peak->yf, obj->x, obj->y); + if (!src) NEXT1; if (!src->peak) NEXT1; @@ -209,10 +245,10 @@ src = sources->data[I]; - if (!src) NEXT1; - if (!src->peak) NEXT1; - if (!isfinite(src->peak->xf)) NEXT1; - if (!isfinite(src->peak->yf)) NEXT1; - if (!isfinite(src->peak->detValue)) NEXT1; - if (sqrt(src->peak->detValue) < MIN_SN) NEXT1; + if (!src) continue; + if (!src->peak) continue; + if (!isfinite(src->peak->xf)) continue; + if (!isfinite(src->peak->yf)) continue; + if (!isfinite(src->peak->detValue)) continue; + if (sqrt(src->peak->detValue) < MIN_SN) continue; dx = src->peak->xf - obj->x; Index: /branches/eam_branches/ipp-20111122/ppSub/src/ppSubReadoutPhotometry.c =================================================================== --- /branches/eam_branches/ipp-20111122/ppSub/src/ppSubReadoutPhotometry.c (revision 32851) +++ /branches/eam_branches/ipp-20111122/ppSub/src/ppSubReadoutPhotometry.c (revision 32852) @@ -92,6 +92,6 @@ // detections in the wings (or cores) of bright(er) stars found in both images. // flag detections based on their distance from the bright(er) input sources. - bool matchRef = !strcasecmp(name, "PPSUB.INVERSE"); - ppSubFlagNeighbors (config, view, sources, matchRef); + bool subInverse = !strcasecmp(name, "PPSUB.INVERSE"); + ppSubFlagNeighbors (config, view, sources, subInverse); if (data->stats) { Index: /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_checkdependent.pl (revision 32851) +++ /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_checkdependent.pl (revision 32852) @@ -255,5 +255,5 @@ # caller will fault the jobs return $error_code; - } elsif ($chip->{dsRun_state} eq 'failed_revert') { + } elsif ($chip->{dsRun_state} =~ /failed_revert/) { # XXX: revert failures are rarely fixed. give up but say it's just not available not GONE print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n"; Index: /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_cleanup.pl =================================================================== --- /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_cleanup.pl (revision 32851) +++ /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_cleanup.pl (revision 32852) @@ -23,4 +23,5 @@ use PS::IPP::Config qw( :standard ); +use PS::IPP::PStamp::Job qw( :standard ); my $req_id; @@ -95,4 +96,22 @@ } +{ + my $command = "$pstamptool -listfile -req_id $req_id"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + die("Unable to perform $command error code: $error_code"); + } + my $output = join "", @$stdout_buf; + if ($output) { + my $files = parse_md_fast($mdcParser, $output); + foreach my $file (@$files) { + $ipprc->file_delete($file->{path}); + } + } +} + # now go find the workdir for this request # XXX: we finally *have* to store this in the database Index: /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_get_image_job.pl =================================================================== --- /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_get_image_job.pl (revision 32851) +++ /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_get_image_job.pl (revision 32852) @@ -25,7 +25,8 @@ my $output_base; +my $bundleroot; my $verbose; -my $ipprc; +my $imagedbname; my $dbname; my $dbserver; @@ -41,4 +42,6 @@ 'rownum=s' => \$rownum, 'output_base=s' => \$output_base, + 'bundleroot=s' => \$bundleroot, + 'imagedbname=s' => \$imagedbname, 'dbname=s' => \$dbname, 'dbserver=s' => \$dbserver, @@ -52,4 +55,6 @@ my_die( $err, $PS_EXIT_PROG_ERROR) if $err; + +my $ipprc = PS::IPP::Config->new(); my $params_file = $output_base . ".mdc"; @@ -91,4 +96,5 @@ my $missing_tools; my $dist_bundle = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1); +my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -96,8 +102,23 @@ } +$pstamptool .= " -dbname $dbname" if $dbname; +$pstamptool .= " -dbserver $dbserver" if $dbserver; + my $outdir = dirname($output_base); my $basename = basename($path_base); -my $outroot = $output_base ."_" . $basename; my $results_file = $output_base . ".bundle_results"; +my $outroot; +if ($bundleroot) { + my (undef, undef, undef, $mday, $month, $year) = gmtime(time()); + $month += 1; + $year += 1900; + + # This will generate an outroot like: + # neb://any/pstamp/bundles/2011/11/22/14026916_1_1_o5739g0358o.353654.wrp.215092.skycell.2083.025 + + $outroot = $bundleroot . sprintf("/%4d/%02d/%02d/${job_id}_", $year, $month, $mday) . basename($output_base) . "_". $basename; +} else { + $outroot = $output_base ."_" . $basename; +} { @@ -105,8 +126,8 @@ $command .= " --results_file $results_file"; $command .= " --component $component --path_base $path_base --outroot $outroot"; -# XXX: we need to do some work if we want to support muggle bundles -# $command .= " --no_magic if $no_magic"; + # XXX: we need to do some work if we want to support muggle bundles + # $command .= " --no_magic if $no_magic"; $command .= " --magicked" if $magicked; - $command .= " --dbname $dbname" if $dbname; + $command .= " --dbname $imagedbname" if $imagedbname; $command .= " --verbose" if $verbose; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -147,4 +168,36 @@ } +if ($bundleroot) { + { + # delete any existing pstampFile in case this job has faulted and + # been reverted + my $command = "$pstamptool -deletefile -job_id $job_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("Unable to perform $command: $error_code", $error_code >> 8); + } + } + my $linkname = "$outdir/$file_name"; + if (-l $linkname or -e $linkname) { + unlink $linkname or my_die("Failed to unlink existing symlink: $linkname", $PS_EXIT_UNKNOWN_ERROR); + } + my $bundle_name = dirname($outroot) . "/$file_name"; + my $resolved = $ipprc->file_resolve($bundle_name); + if (!$resolved) { + my_die("failed to resolve $bundle_name", $PS_EXIT_UNKNOWN_ERROR); + } + symlink $resolved, $linkname or my_die("failed to create symlink to $resolved in $outdir", + $PS_EXIT_UNKNOWN_ERROR); + + my $command = "$pstamptool -addfile -job_id $job_id -path $bundle_name"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("Unable to perform $command: $error_code", $error_code >> 8); + } +} + + print REGLIST "$file_name|$bytes|$md5sum|tgz|\n"; Index: /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_job_run.pl (revision 32851) +++ /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_job_run.pl (revision 32852) @@ -88,7 +88,8 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $params = read_params_file($outputBase); + my $jobStatus; if ($jobType eq "stamp") { - my $params = read_params_file($outputBase); my $argString; @@ -272,6 +273,10 @@ } elsif ($jobType eq "get_image") { - my $uri = ""; + my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT"); + my $imagedb = $params->{imagedb}; + my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum"; + $command .= " --bundleroot $pstamp_bundle_root" if $pstamp_bundle_root; + $command .= " --imagedb $imagedb" if $imagedb; $command .= " --dbname $dbname" if $dbname; $command .= " --dbserver $dbserver" if $dbserver; Index: /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_parser_run.pl (revision 32851) +++ /branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_parser_run.pl (revision 32852) @@ -89,9 +89,14 @@ # outdir is where all of the files generated for this request are placed # NOTE: this location needs to be kept in sync with the web interface ( request.php ) - my $datestr = strftime "%Y%m%d", gmtime; + my $datestr = strftime "%Y/%m/%d", gmtime; my $datedir = "$pstamp_workdir/$datestr"; if (! -e $datedir ) { - mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id, - $PS_EXIT_CONFIG_ERROR); + my $rc = system "mkdir -p $datedir"; + if ($rc) { + my $status = $rc >> 8; + $status = $PS_EXIT_CONFIG_ERROR if !$status; + my_die( "failed to create working directory $datedir for request id $req_id", $req_id, + $status); + } } Index: /branches/eam_branches/ipp-20111122/psvideophot/src/psvideoLoop.c =================================================================== --- /branches/eam_branches/ipp-20111122/psvideophot/src/psvideoLoop.c (revision 32851) +++ /branches/eam_branches/ipp-20111122/psvideophot/src/psvideoLoop.c (revision 32852) @@ -234,5 +234,7 @@ Centering(readout->image, EDGE, &vpSum, xy, median); - vpFlux = SubSum(readout->image, xy, median); + if (isfinite(xy[0]) && isfinite(xy[1])) { + vpFlux = SubSum(readout->image, xy, median); + } // printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.2f\n",