IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 32852


Ignore:
Timestamp:
Dec 2, 2011, 11:45:13 AM (15 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20111122
Files:
22 edited
3 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20111122/dbconfig/add.md

    r31378 r32852  
    33    add_id          S64     0       # Primary Key AUTO_INCREMENT
    44    stage           STR     64      # what the stage is (warp, cam, diff,
    5     stage_id          S64     0       # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id)
     5    stage_id          S64     0       # Key INDEX(add_id,cam_id) fkey(cam_id)    ref camRun(cam_id)
     6    stage_extra1    S32     0       
    67    state           STR     64      # Key
    78    workdir         STR     255

  • branches/eam_branches/ipp-20111122/dbconfig/changes.txt

    r32609 r32852  
    21772177
    21782178UPDATE dbversion set schema_version = '1.1.71', updated= CURRENT_TIMESTAMP();
     2179
     2180
     2181-- this is to stuff extra stage related information.
     2182-- example - multi filter staticsky stage - this allows selection of
     2183-- each filter of cmf staticsky. 
     2184
     2185alter table addRun add column stage_extra1 int after stage_id;

  • branches/eam_branches/ipp-20111122/dbconfig/pstamp.md

    r30314 r32852  
    6767    num         S64         0    # Primary Key AUTO_INCREMENT
    6868END
     69
     70pstampFile METADATA
     71    file_id     S64         0   # Primary Key AUTO_INCREMENT
     72    job_id      S64         0
     73    path        STR         255
     74end

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py

    r32596 r32852  
    286286    Publishes this batch to the datastore
    287287    '''
    288     def tarZipAndpublishToDatastore(self):
     288    def tarAndZip(self):
    289289     
    290290        # set up filenams and paths
    … …  
    295295
    296296        # tar directory
    297         p = Popen("tar -cvf " + tarPath + "\
    298                 -C " + self.subDir + " \
    299                 " + self.batchName, shell=True, stdout=PIPE)
     297        cmd = "tar -cvf " + tarPath + " -C " + self.subDir + " " + self.batchName
     298        self.logger.infoPair("Creating tar archive", cmd)
     299        p = Popen(cmd, shell=True, stdout=PIPE)
    300300        p.wait()
    301301
     302        if p.returncode != 0:
     303            self.logger.errorPair("tar command", "failed")
     304            return False
     305
    302306        # zip tar archive
    303         p = Popen("gzip -c " + tarPath + " > " + tarballPath, shell=True, stdout=PIPE)
     307        cmd = "gzip -c " + tarPath + " > " + tarballPath
     308        self.logger.infoPair("Compressing tar archive", cmd)
     309        p = Popen(cmd, shell=True, stdout=PIPE)
    304310        p.wait()
     311
     312        if p.returncode != 0:
     313            self.logger.errorPair("gzip command", "failed")
     314            return False
     315
     316        # only now can we report that the batch has successfully processed
     317        self.ippToPspsDb.updateProcessed(self.batchID, 1)
    305318
    306319        # delete tar file and original directory
    … …  
    308321        shutil.rmtree(self.localOutPath)
    309322
    310         # now publish to the datastore
    311         Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile)
     323        return True
    312324
    313325    '''
    … …  
    448460        try:
    449461            stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
    450             self.ippToPspsDb.updateProcessed(self.batchID, 1)
    451462        except:
    452463            self.logger.exception("Could not write to FITS")
    … …  
    494505        p = Popen(cmd, shell=True, stdout=PIPE)
    495506        p.wait()
    496         #        out = p.stdout.read()
    497507
    498508        rowCount = self.scratchDb.getRowCount("dvoDetection")
    … …  
    507517    '''
    508518    Creates and publishes a batch
    509     TODO all methdds call below should throw exceptions on failure
     519    TODO all methods call below should throw exceptions on failure
    510520    '''
    511521    def run(self):
    … …  
    530540            else:
    531541                self.writeBatchManifest()
    532                 if int(self.doc.find("options/publishToDatastore").text): self.tarZipAndpublishToDatastore()
     542                if int(self.doc.find("options/publishToDatastore").text):
     543                    # tar and zip ready for publication to datastore
     544                    if self.tarAndZip():
     545                        # now publish to the datastore
     546                        tarballFile = Batch.getTarballFile(self.batchID)
     547                        Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile)
     548
    533549                if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False)
    534550

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi.xml

    r32736 r32852  
    4848    <gpc1Label>ThreePi.V3</gpc1Label>
    4949    <location>/data/ipp005.0/gpc1/catdirs/ThreePi.V3</location>
    50     <minRA>135</minRA>
     50    <minRA>105</minRA>
    5151    <maxRA>270</maxRA>
    5252    <minDec>-100</minDec>

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/datastore.py

    r32491 r32852  
    4848        # build dsreg command
    4949        command  = "dsreg --add " + name + "\
    50                     --copy --datapath " + path + "\
     50                    --link --datapath " + path + "\
    5151                    --type " + self.type + "\
    5252                    --product " + self.product + "\

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py

    r32493 r32852  
    3030        self.logger.debug("Gpc1Db destructor")
    3131
    32 
    33     '''
    34     TODO
    35     '''
    36     def getIDsInThisDVODbForThisStageFudge(self):
    37 
    38         sql = "SELECT staticskyRun.sky_id \
    39                FROM staticskyInput, staticskyRun, stackRun, staticskyResult \
    40                WHERE staticskyRun.sky_id = staticskyInput.sky_id \
    41                AND staticskyInput.stack_id = stackRun.stack_id \
    42                AND staticskyInput.sky_id = staticskyResult.sky_id \
    43                AND staticskyRun.label like 'MD04.staticsky' \
    44                AND stackRun.filter like 'i%'"
    45 
    46         try:
    47             rs = self.executeQuery(sql)
    48         except:
    49             self.logger.exception("Can't query for ids in DVO")
    50 
    51         ids = []
    52         while (rs.next()):
    53             ids.append(rs.getInt(1))
    54 
    55         rs.close()
    56 
    57         self.logger.debug("Found %d items in DVO database '" % (len(ids)))
    58 
    59         return ids
    6032
    6133    '''

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopspsdb.py

    r32654 r32852  
    103103    '''
    104104    Returns a list of processed batch IDs that are not loaded to the datastore
     105    NB we have a date constraint here because, before this time, it is possible that
     106    a test batch would be labelled as processed but not loaded to datastore, and we would NOT
     107    want to load a test batch to PSPS
     108
     109    We also check that these batches are more than 4 HOURs old, so that we don't attempt to
     110    publish something simulataneously with the client that it producing it
    105111    '''
    106112    def getProcessedButFailedDatastoreBatchIDs(self, epoch, dvoGpc1Label, batchType):
    … …  
    112118               AND dvo_db = '" + dvoGpc1Label + "' \
    113119               AND processed = 1\
    114                AND loaded_to_datastore = -1"
     120               AND loaded_to_datastore != 1 \
     121               AND timestamp > '2011-10-27' \
     122               AND timestamp < now() -  INTERVAL 4 HOUR"
    115123
    116124        ids = []
    … …  
    466474               AND batch_type = '" + batchType + "' \
    467475               AND dvo_db = '" + dvoGpc1Label + "' \
    468                AND timestamp BETWEEN now() - INTERVAL 2 HOUR AND now()"
     476               AND timestamp BETWEEN now() - INTERVAL 4 HOUR AND now()"
    469477
    470478        try:
    … …  
    499507
    500508    '''
    501     Have we already processed and published this batch?
     509    Have we already processed this stage_id?
    502510    '''
    503511    def alreadyProcessed(self, batchType, stage_id, epoch, dvoGpc1Label):
    … …  
    509517               AND timestamp > '" + epoch + "' \
    510518               AND dvo_db = '" + dvoGpc1Label + "' \
    511                AND processed = 1 \
    512                AND loaded_to_datastore != 0"
     519               AND processed = 1"
    513520
    514521        try:
    … …  
    655662        self.execute(sql)
    656663
     664    '''
     665    Resets a batch ready for re-loading to the datastore
     666    '''
     667    def resetBatch(self, batchID):
     668
     669        sql = "UPDATE batch SET \
     670               loaded_to_datastore = 0, \
     671               loaded_to_ODM = 0, \
     672               deleted_datastore = 0, \
     673               deleted_dxlayer = 0, \
     674               comment = 'This batch was reset, ready for re-load to datastore' \
     675               WHERE batch_id = " + str(batchID)
     676
     677        self.execute(sql)
    657678
    658679    '''

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/metrics.py

    r32587 r32852  
    11import math
     2import time
    23import logging
    34import smtplib
    … …  
    1011from pslogger import PSLogger
    1112from ipptopspsdb import IppToPspsDb
     13from czardb import CzarDb
    1214from gpc1db import Gpc1Db
    1315
    … …  
    7678       numPending = len(pending)
    7779
    78        #if stage == 'merge_worthy' and batchType == "P2":
    79        #    for n in pending: print n
     80       #if stage == 'loaded_to_datastore' and batchType == "P2":
     81       #    for n in fail: print n
    8082
    8183       str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (getIntAsString(numPending), getIntAsString(numSuccess), getIntAsString(numFail))
    … …  
    8486       prevList = success
    8587       print >> DATFILE, stage, numSuccess, numFail, numPending
     88
     89       czarDb.insertStats(stage, DVOLABEL, batchType, numPending, numSuccess, numFail)
    8690
    8791       if stage == 'processed': numPendingProcessed = numPending
    … …  
    124128if len(sys.argv) > 1: CONFIG  = sys.argv[1]
    125129else:
    126     print "** Usage: " + sys.argv[0] + " <configPath>"
     130    print "** Usage: " + sys.argv[0] + " <configPath> [hours]"
    127131    sys.exit(1)
    128132
    129 now = datetime.datetime.now()
     133if len(sys.argv) > 2:
     134    HOURS = float(sys.argv[2])
     135    SECONDS = HOURS * 60.0 * 60.0
     136else:
     137    SECONDS = None
     138
     139
    130140
    131141# open config file
    … …  
    136146logger.setup(configDoc, "metrics")
    137147
     148# create database objects
    138149ippToPspsDb = IppToPspsDb(logger, configDoc)
     150czarDb = CzarDb(logger, configDoc)
    139151gpc1Db = Gpc1Db(logger, configDoc)
    140 
    141 MINRA = None
    142 MAXRA = None
    143 MINDEC = None
    144 MAXDEC = None
    145152
    146153DVOLABEL = configDoc.find("dvo/gpc1Label").text
    147154EPOCH = configDoc.find("options/epoch").text
    148155
    149 logger.infoTitle("ippToPsps loading summary")
    150 logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
    151 logger.infoPair("Loading epoch", EPOCH)
    152 logger.infoPair("DVO label", DVOLABEL)
    153 
    154 try:
    155     MINRA = float(configDoc.find("dvo/minRA").text)
    156     MAXRA = float(configDoc.find("dvo/maxRA").text)
    157     MINDEC = float(configDoc.find("dvo/minDec").text)
    158     MAXDEC = float(configDoc.find("dvo/maxDec").text)
    159     logger.infoPair("Coordinates", "%.1f to %.1f RA, %.1f to %.1f Dec" % (MINRA, MAXRA, MINDEC, MAXDEC))
    160 except:
    161     pass
     156while True:
     157
     158    logger.infoTitle("ippToPsps loading summary")
     159    now = datetime.datetime.now()
     160    logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
     161    logger.infoPair("Loading epoch", EPOCH)
     162    logger.infoPair("DVO label", DVOLABEL)
     163
     164    # get equatorial coord constraints, if any
     165    MINRA = None
     166    MAXRA = None
     167    MINDEC = None
     168    MAXDEC = None
     169
     170    try:
     171        MINRA = float(configDoc.find("dvo/minRA").text)
     172        MAXRA = float(configDoc.find("dvo/maxRA").text)
     173        MINDEC = float(configDoc.find("dvo/minDec").text)
     174        MAXDEC = float(configDoc.find("dvo/maxDec").text)
     175        logger.infoPair("Coordinates", "%.1f to %.1f RA, %.1f to %.1f Dec" % (MINRA, MAXRA, MINDEC, MAXDEC))
     176    except:
     177        pass
    162178   
    163179
    164 logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    165 logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |")
    166 logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    167 rateP2 = printStats("P2")
    168 rateST = printStats("ST")
    169 logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    170 
    171 logger.info("")
    172 
    173 stages = ippToPspsDb.getStages()
    174 colCount = len(stages)
    175 writetableSeparator(colCount)
    176 sys.stdout.write("|Type|  DVO  ")
    177 for stage in stages: sys.stdout.write("|%19s" % stage)
    178 sys.stdout.write("|\n")
    179 sys.stdout.write("|    |       ")
    180 for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
    181 sys.stdout.write("|\n")
    182 writetableSeparator(colCount)
    183 pendP2Processed = printTableRow(stages, "P2")
    184 pendSTProcessed = printTableRow(stages, "ST")
    185 writetableSeparator(colCount)
    186 
    187 logger.info("")
    188 try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2))
    189 except: pass
    190 try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST))
    191 except: pass
     180    logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     181    logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |")
     182    logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     183    rateP2 = printStats("P2")
     184    rateST = printStats("ST")
     185    logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     186
     187    logger.info("")
     188
     189    stages = ippToPspsDb.getStages()
     190    colCount = len(stages)
     191    writetableSeparator(colCount)
     192    sys.stdout.write("|Type|  DVO  ")
     193
     194    for stage in stages: sys.stdout.write("|%19s" % stage)
     195    sys.stdout.write("|\n")
     196    sys.stdout.write("|    |       ")
     197    for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
     198    sys.stdout.write("|\n")
     199    writetableSeparator(colCount)
     200    pendP2Processed = printTableRow(stages, "P2")
     201    pendSTProcessed = printTableRow(stages, "ST")
     202    writetableSeparator(colCount)
     203
     204    logger.info("")
     205    try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2))
     206    except: pass
     207    try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST))
     208    except: pass
     209
     210    if SECONDS:
     211        logger.infoPair("Waiting for", "%f hours" % HOURS)
     212        time.sleep(SECONDS)
     213    else: break
     214
     215

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/odm.py

    r32540 r32852  
    6767            STATE = doc.find("{%s}OdmBatchState/{%s}BatchState" % (NAMESPACE, NAMESPACE)).text
    6868            MESSAGE = doc.find("{%s}OdmBatchState/{%s}Message" % (NAMESPACE, NAMESPACE)).text
     69            results['MESSAGE'] = MESSAGE
     70            results['DETAILS'] = "None"
    6971
    7072            # we assume that batches 'OnHold' are failed as it is very rare for such

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/republishbatch.py

    r32497 r32852  
    1111
    1212'''
     13A program that republishes a bath to the datastore.
    1314
     15It first 'resets' the batch in the ippToPsps database, i.e. pretends it has never been published, then re-publishes it to the datastore
    1416'''
    1517if len(sys.argv) > 2:
    … …  
    5052logger.infoPair("subdir", subDir)
    5153logger.infoPair("tarballfile", tarballFile)
     54logger.infoPair("Reseting batch in database", "%d" % BATCHID)
     55ippToPspsDb.resetBatch(BATCHID)
    5256logger.infoPair("publishing to", DATASTOREPRODUCT)
    5357Batch.publishToDatastore(datastore, BATCHID, batchName, subDir, tarballFile)

  • branches/eam_branches/ipp-20111122/ippToPsps/xmlschema/BatchManifest.xsd

    r29075 r32852  
    5151    <xs:enumeration value="SSS" />
    5252    <xs:enumeration value="3PI" />
    53     <xs:enumeration value="STS1" />
     53    <xs:enumeration value="STS" />
     54    <xs:enumeration value="SA1" />
     55    <xs:enumeration value="SA2" />
     56    <xs:enumeration value="SA3" />
     57    <xs:enumeration value="SAS" />
     58    <xs:enumeration value="OLD" />
    5459   </xs:restriction>
    5560  </xs:simpleType>

  • branches/eam_branches/ipp-20111122/ppSub/src/ppSubFlagNeighbors.c

    r31449 r32852  
    1818#include "ppSub.h"
    1919
     20# define MIN_ERROR 0.02 /* padding for self-match flux test */
     21# define MAX_OFFSET 5.0 /* delta mag greater than this is not a self match */
     22
    2023// match the diff sources to the detections from the positive or negative input images
    2124// matchRef : false if this is the standard diff image, true if this is the inverse (in-ref vs ref-in)
    2225// this is needed to choose which value is positive and which is negative relative to the difference
    23 bool ppSubFlagNeighbors(pmConfig *config, pmFPAview *view, psArray *sources, bool matchRef) {
     26bool ppSubFlagNeighbors(pmConfig *config, pmFPAview *view, psArray *sources, bool subInverse) {
    2427
    2528    bool status;
    … …  
    106109              psAssert ((sourceM1->imageID == 1) || (sourceM1->imageID == 2), "error in pmPhotObj construction (second entry is not from 1 or 2");
    107110
     111              // which image gives the match, the positive or negative one?
     112              bool positive = false;
     113              if (subInverse) {
     114                  positive = (sourceM1->imageID == 2);
     115              } else {
     116                  positive = (sourceM1->imageID == 1);
     117              }
     118
     119              // check if source and sourceM1 are likely to be the same flux (ie, isolated detection in one image)
     120              if (positive && isfinite(sourceM1->psfMag) && isfinite(source->psfMag)) {
     121                  float dMag = sourceM1->psfMag - source->psfMag;
     122                  float nSig = fabs(dMag) / hypot(MIN_ERROR, source->psfMagErr);
     123                  if (nSig < MAX_OFFSET) {
     124                      source->mode2 |= PM_SOURCE_MODE2_DIFF_SELF_MATCH;
     125                  }
     126              }
     127
    108128              // only one match.  set a flag?
    109129              source->mode2 |= PM_SOURCE_MODE2_DIFF_WITH_SINGLE;
    110130              if (source->diffStats) {
    111                   // which is match need an xor here...
    112                   bool positive = !matchRef && (sourceM1->imageID == 1);
    113                   positive |= matchRef && (sourceM1->imageID == 2);
    114131                  float SN1 = isfinite(sourceM1->psfMagErr) ? 1.0 / sourceM1->psfMagErr : NAN;
    115132                  if (positive) {
    … …  
    133150              psAssert (sourceM1->imageID != sourceM2->imageID, "error in pmPhotObj construction (case 3.3)");
    134151
     152              // we don't know which matched source (sourceM1 or sourceM2) is the positive detection,
     153              // and the answer depends on both the ID and the subtraction order (in-ref vs ref-in)
     154              // positive is true if sourceM1 is the positive detection
     155              bool positive = false;
     156              if (subInverse) {
     157                  positive = (sourceM1->imageID == 2);
     158              } else {
     159                  positive = (sourceM1->imageID == 1);
     160              }
     161
     162              // check for self-detection
     163              float magPSF1 = positive ? sourceM1->psfMag : sourceM2->psfMag;
     164              if (isfinite(magPSF1) && isfinite(source->psfMag)) {
     165                  float dMag = magPSF1 - source->psfMag;
     166                  float nSig = fabs(dMag) / hypot(MIN_ERROR, source->psfMagErr);
     167                  if (nSig < MAX_OFFSET) {
     168                      source->mode2 |= PM_SOURCE_MODE2_DIFF_SELF_MATCH;
     169                  }
     170              }
     171
    135172              source->mode2 |= PM_SOURCE_MODE2_DIFF_WITH_DOUBLE;
    136173              if (source->diffStats) {
    137                   // which is match need an xor here...
    138                   bool positive = !matchRef && (sourceM1->imageID == 1);
    139                   positive |= matchRef && (sourceM1->imageID == 2);
    140174                  float SN1 = isfinite(sourceM1->psfMagErr) ? 1.0 / sourceM1->psfMagErr : NAN;
    141175                  float SN2 = isfinite(sourceM2->psfMagErr) ? 1.0 / sourceM2->psfMagErr : NAN;
    … …  
    187221        pmPhotObj *obj = objects->data[j];
    188222 
     223        // fprintf (stderr, "%f, %f vs %f, %f\n", src->peak->xf, src->peak->yf, obj->x, obj->y);
     224
    189225        if (!src) NEXT1;
    190226        if (!src->peak) NEXT1;
    … …  
    209245            src = sources->data[I];
    210246           
    211             if (!src) NEXT1;
    212             if (!src->peak) NEXT1;
    213             if (!isfinite(src->peak->xf)) NEXT1;
    214             if (!isfinite(src->peak->yf)) NEXT1;
    215             if (!isfinite(src->peak->detValue)) NEXT1;
    216             if (sqrt(src->peak->detValue) < MIN_SN) NEXT1;
     247            if (!src) continue;
     248            if (!src->peak) continue;
     249            if (!isfinite(src->peak->xf)) continue;
     250            if (!isfinite(src->peak->yf)) continue;
     251            if (!isfinite(src->peak->detValue)) continue;
     252            if (sqrt(src->peak->detValue) < MIN_SN) continue;
    217253
    218254            dx = src->peak->xf - obj->x;

  • branches/eam_branches/ipp-20111122/ppSub/src/ppSubReadoutPhotometry.c

    r31449 r32852  
    9292    // detections in the wings (or cores) of bright(er) stars found in both images.
    9393    // flag detections based on their distance from the bright(er) input sources.
    94     bool matchRef = !strcasecmp(name, "PPSUB.INVERSE");
    95     ppSubFlagNeighbors (config, view, sources, matchRef);
     94    bool subInverse = !strcasecmp(name, "PPSUB.INVERSE");
     95    ppSubFlagNeighbors (config, view, sources, subInverse);
    9696
    9797    if (data->stats) {

  • branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_checkdependent.pl

    r32364 r32852  
    255255            # caller will fault the jobs
    256256            return $error_code;
    257         } elsif ($chip->{dsRun_state} eq 'failed_revert') {
     257        } elsif ($chip->{dsRun_state} =~ /failed_revert/) {
    258258            # XXX: revert failures are rarely fixed. give up but say it's just not available not GONE
    259259            print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n";

  • branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_cleanup.pl

    r27874 r32852  
    2323
    2424use PS::IPP::Config qw( :standard );
     25use PS::IPP::PStamp::Job qw( :standard );
    2526
    2627my $req_id;
    … …  
    9596}
    9697
     98{
     99    my $command = "$pstamptool -listfile -req_id $req_id";
     100    $command   .= " -dbname $dbname" if $dbname;
     101    $command   .= " -dbserver $dbserver" if $dbserver;
     102    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     103        run(command => $command, verbose => $verbose);
     104    unless ($success) {
     105        die("Unable to perform $command error code: $error_code");
     106    }
     107    my $output = join "", @$stdout_buf;
     108    if ($output) {
     109        my $files = parse_md_fast($mdcParser, $output);
     110        foreach my $file (@$files) {
     111            $ipprc->file_delete($file->{path});
     112        }
     113    }
     114}
     115
    97116# now go find the workdir for this request
    98117# XXX: we finally *have* to store this in the database

  • branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_get_image_job.pl

    r30663 r32852  
    2525
    2626my $output_base;
     27my $bundleroot;
    2728
    2829my $verbose;
    29 my $ipprc;
     30my $imagedbname;
    3031my $dbname;
    3132my $dbserver;
    … …  
    4142        'rownum=s'        =>      \$rownum,
    4243        'output_base=s'   =>      \$output_base,
     44        'bundleroot=s'    =>      \$bundleroot,
     45        'imagedbname=s'   =>      \$imagedbname,
    4346        'dbname=s'        =>      \$dbname,
    4447        'dbserver=s'      =>      \$dbserver,
    … …  
    5255
    5356my_die( $err, $PS_EXIT_PROG_ERROR) if $err;
     57
     58my $ipprc = PS::IPP::Config->new();
    5459
    5560my $params_file = $output_base . ".mdc";
    … …  
    9196my $missing_tools;
    9297my $dist_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
     98my $pstamptool   = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
    9399if ($missing_tools) {
    94100    warn("Can't find required tools.");
    … …  
    96102}
    97103
     104$pstamptool .= " -dbname $dbname" if $dbname;
     105$pstamptool .= " -dbserver $dbserver" if $dbserver;
     106
    98107my $outdir = dirname($output_base);
    99108my $basename = basename($path_base);
    100 my $outroot = $output_base ."_" . $basename;
    101109my $results_file = $output_base . ".bundle_results";
     110my $outroot;
     111if ($bundleroot) {
     112    my (undef, undef, undef, $mday, $month, $year) = gmtime(time());
     113    $month += 1;
     114    $year += 1900;
     115
     116    # This will generate an outroot like:
     117    # neb://any/pstamp/bundles/2011/11/22/14026916_1_1_o5739g0358o.353654.wrp.215092.skycell.2083.025
     118
     119    $outroot = $bundleroot . sprintf("/%4d/%02d/%02d/${job_id}_", $year, $month, $mday) . basename($output_base) . "_". $basename;
     120} else {
     121    $outroot = $output_base ."_" . $basename;
     122}
    102123
    103124{
    … …  
    105126    $command .= " --results_file $results_file";
    106127    $command .= " --component $component --path_base $path_base --outroot $outroot";
    107 #    XXX: we need to do some work if we want to support muggle bundles
    108 #    $command .= " --no_magic if $no_magic";
     128    #    XXX: we need to do some work if we want to support muggle bundles
     129    #    $command .= " --no_magic if $no_magic";
    109130    $command .= " --magicked" if $magicked;
    110     $command .= " --dbname $dbname" if $dbname;
     131    $command .= " --dbname $imagedbname" if $imagedbname;
    111132    $command .= " --verbose" if $verbose;
    112133    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    147168}
    148169
     170if ($bundleroot) {
     171    {
     172        # delete any existing pstampFile in case this job has faulted and
     173        # been reverted
     174        my $command = "$pstamptool -deletefile -job_id $job_id";
     175        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     176            run(command => $command, verbose => $verbose);
     177        unless ($success) {
     178            my_die("Unable to perform $command: $error_code", $error_code >> 8);
     179        }
     180    }
     181    my $linkname = "$outdir/$file_name";
     182    if (-l $linkname or -e $linkname) {
     183        unlink $linkname or my_die("Failed to unlink existing symlink: $linkname", $PS_EXIT_UNKNOWN_ERROR);
     184    }
     185    my $bundle_name = dirname($outroot) . "/$file_name";
     186    my $resolved = $ipprc->file_resolve($bundle_name);
     187    if (!$resolved) {
     188        my_die("failed to resolve $bundle_name", $PS_EXIT_UNKNOWN_ERROR);
     189    }
     190    symlink $resolved, $linkname or my_die("failed to create symlink to $resolved in $outdir",
     191        $PS_EXIT_UNKNOWN_ERROR);
     192
     193    my $command = "$pstamptool -addfile -job_id $job_id -path $bundle_name";
     194    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     195        run(command => $command, verbose => $verbose);
     196    unless ($success) {
     197        my_die("Unable to perform $command: $error_code", $error_code >> 8);
     198    }
     199}
     200
     201
    149202print REGLIST "$file_name|$bytes|$md5sum|tgz|\n";
    150203

  • branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_job_run.pl

    r31508 r32852  
    8888my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    8989
     90my $params = read_params_file($outputBase);
     91
    9092my $jobStatus;
    9193if ($jobType eq "stamp") {
    92     my $params = read_params_file($outputBase);
    9394
    9495    my $argString;
    … …  
    272273} elsif ($jobType eq "get_image") {
    273274
    274     my $uri = "";
     275    my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT");
     276    my $imagedb = $params->{imagedb};
     277
    275278    my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum";
     279    $command .= " --bundleroot $pstamp_bundle_root" if $pstamp_bundle_root;
     280    $command .= " --imagedb $imagedb" if $imagedb;
    276281    $command .= " --dbname $dbname" if $dbname;
    277282    $command .= " --dbserver $dbserver" if $dbserver;

  • branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_parser_run.pl

    r30317 r32852  
    8989    # outdir is where all of the files generated for this request are placed
    9090    # NOTE: this location needs to be kept in sync with the web interface ( request.php )
    91     my $datestr = strftime "%Y%m%d", gmtime;
     91    my $datestr = strftime "%Y/%m/%d", gmtime;
    9292    my $datedir = "$pstamp_workdir/$datestr";
    9393    if (! -e $datedir ) {
    94         mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
    95             $PS_EXIT_CONFIG_ERROR);
     94        my $rc = system "mkdir -p $datedir";
     95        if ($rc) {
     96            my $status = $rc >> 8;
     97            $status = $PS_EXIT_CONFIG_ERROR if !$status;
     98            my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
     99                $status);
     100        }
    96101    }
    97102

  • branches/eam_branches/ipp-20111122/psvideophot

  • branches/eam_branches/ipp-20111122/psvideophot/src

    • Property svn:ignore

      •  

        old new  
        11psvideophot
         2dumpvideo
        23Makefile
        34Makefile.in
        … …  
        910psvideophotVersionDefinitions.h
        1011
         12

  • branches/eam_branches/ipp-20111122/psvideophot/src/psvideoLoop.c

    r32556 r32852  
    234234                    Centering(readout->image, EDGE, &vpSum, xy, median);
    235235
    236                     vpFlux = SubSum(readout->image, xy, median);
     236                    if (isfinite(xy[0]) && isfinite(xy[1])) {
     237                        vpFlux = SubSum(readout->image, xy, median);
     238                    }
    237239
    238240//                    printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.2f\n",
Note: See TracChangeset for help on using the changeset viewer.