IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 35417


Ignore:
Timestamp:
Apr 19, 2013, 4:35:54 PM (13 years ago)
Author:
eugene
Message:

use MEMORY engine for dvoDetectionFull; add error handline for dvopsps; check mysql version; remove some test verbosity; check for image existence in dvoImages; skip bad batches but do not exit; fix precision issues; add -realgpc option (part of -test mode); log times in milliseconds

Location:
trunk/ippToPsps
Files:
21 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps

  • trunk/ippToPsps/config/settings.xml

    r35217 r35417  
    1515      <user>dvo</user>
    1616      <password>dvo</password>
     17      <memory>16</memory>
    1718    </loader>
    1819    <loader mach="ipp007">
    … …  
    2122      <user>dvo</user>
    2223      <password>dvo</password>
     24      <memory>16</memory>
    2325    </loader>
    2426    <loader mach="ipp008">
    … …  
    2729      <user>dvo</user>
    2830      <password>dvo</password>
     31      <memory>16</memory>
    2932    </loader>
    3033    <loader mach="ipp009">
    … …  
    3336      <user>dvo</user>
    3437      <password>dvo</password>
     38      <memory>16</memory>
    3539    </loader>
    3640    <loader mach="ipp010">
    … …  
    3943      <user>dvo</user>
    4044      <password>dvo</password>
     45      <memory>16</memory>
     46    </loader>
     47    <loader mach="ipp064">
     48      <name>ipptopsps_scratch</name>
     49      <host>ipp064</host>
     50      <user>dvo</user>
     51      <password>dvo</password>
     52      <memory>32</memory>
    4153    </loader>
    4254    <loader mach="generic">
    … …  
    4557      <user>dvo</user>
    4658      <password>dvo</password>
     59      <memory>16</memory>
    4760    </loader>
    4861  </localdatabase>
    4962
    5063  <!-- GPC1 Db section (REAL version) -->
     64  <!-- this db is used for real processing, or in a test if -realgpc
     65       is supplied on the cmd line -->
    5166  <gpc1database>
    5267    <name>gpc1</name>
    … …  
    7186      <user>dvo</user>
    7287      <password>dvo</password>
     88      <memory>16</memory>
     89    </loader>
     90    <loader mach="ipp064">
     91      <name>ipptopsps_test_scratch</name>
     92      <host>ipp064</host>
     93      <user>dvo</user>
     94      <password>dvo</password>
     95      <memory>32</memory>
    7396    </loader>
    7497    <loader mach="pikake">
    … …  
    77100      <user>dvo</user>
    78101      <password>dvo</password>
     102      <memory>2</memory>
    79103    </loader>
    80104    <loader mach="generic">
    … …  
    83107      <user>dvo</user>
    84108      <password>dvo</password>
     109      <memory>2</memory>
    85110    </loader>
    86111  </localdatabase_test>
    87112
    88   <!-- GPC1 Db section (REAL VERSION) -->
    89   <gpc1database_test>
    90     <name>gpc1</name>
    91     <host>ippdb01</host>
    92     <user>ipp</user>
    93     <password>ipp</password>
    94   </gpc1database_test>
    95 
    96   <!-- GPC1 Db section (TEST VERSION)
     113  <!-- GPC1 Db section (TEST VERSION)-->
     114  <!-- this db is used for test processing UNLESS -realgpc is supplied
     115       on the cmd line -->
    97116  <gpc1database_test>
    98117    <name>gpc1test</name>
    … …  
    100119    <user>dvo</user>
    101120    <password>dvo</password>
    102   </gpc1database_test> -->
     121  </gpc1database_test>
    103122
    104123  <!-- ippToPsps Db section (TEST VERSION)  -->

  • trunk/ippToPsps/jython/batch.py

    r35174 r35417  
    276276            p = Popen(cmd, shell=True, stdout=PIPE)
    277277            p.wait()
     278            if p.returncode != 0:
     279                self.logger.errorPair("error running xmllint for ", tmpPath)
     280                raise
    278281        except:
    279282            self.logger.errorPair("Popen failed", cmd)
    … …  
    301304        p = Popen(cmd, shell=True, stdout=PIPE)
    302305        p.wait()
    303 
    304306        if p.returncode != 0:
    305307            self.logger.errorPair("tar command", "failed")
    … …  
    311313        p = Popen(cmd, shell=True, stdout=PIPE)
    312314        p.wait()
    313 
    314315        if p.returncode != 0:
    315316            self.logger.errorPair("gzip command", "failed")
    … …  
    415416
    416417          # drop any previous tables before import
    417           #self.scratchDb.dropTable(table.name)
     418          self.scratchDb.dropTable(table.name)
    418419
    419420          # IPP FITS files are littered with infinities, so remove these
    … …  
    426427          try:
    427428              table.write(self.scratchDb.url + '#' + table.name)
    428               # print "wrote to mysql ", table
    429               self.scratchDb.killLastConnectionID()
     429              # XXX this one is not needed
     430              # self.scratchDb.killLastConnectionID()
    430431              count = count + 1
    431432          except:
    … …  
    454455           # get everything from table
    455456           try:
     457               print "reading table from mysql ", table
    456458               _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
    457459               self.scratchDb.killLastConnectionID()
    … …  
    518520        p = Popen(cmd, shell=True, stdout=PIPE)
    519521        p.wait()
     522        if p.returncode != 0:
     523            self.logger.errorPair("dvograbber command", "failed")
     524            return False
    520525
    521526        rowCount = self.scratchDb.getRowCount("dvoDetection")
    … …  
    554559
    555560        if not self.populatePspsTables():
     561            if self.skipBatch:
     562                self.ippToPspsDb.updateProcessed(self.batchID, -1)
     563                return True
     564
    556565            self.logger.errorPair("Aborting this batch", "unable to populate PSPS tables")
    557566            self.ippToPspsDb.updateProcessed(self.batchID, -1)

  • trunk/ippToPsps/jython/config.py

    r35217 r35417  
    2828            if arg == "-t":
    2929                self.test = True
     30                sys.argv.remove(arg)
     31
     32        self.gpc1test = self.test
     33        for arg in sys.argv:
     34            if arg == "-realgpc":
     35                self.gpc1test = False
    3036                sys.argv.remove(arg)
    3137
    … …  
    6066        self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text
    6167
    62 
    63 
    64         print "config.programName: ", self.programName
    65         print "config.configDir: ", self.configDir
    66         print "config.settingsPath: ", self.settingsPath
    67         print "config.logPath: ", self.logPath
    68         print "config.test: ", self.test
    69         print "config.czarPlotsPath: ", self.czarPlotsPath
     68        # print "config.programName: ", self.programName
     69        # print "config.configDir: ", self.configDir
     70        # print "config.settingsPath: ", self.settingsPath
     71        # print "config.logPath: ", self.logPath
     72        # print "config.test: ", self.test
     73        # print "config.czarPlotsPath: ", self.czarPlotsPath
    7074
    7175        root = self.settingsDoc.getroot()
    … …  
    119123            return self.settingsDoc.find(dbType +"/password").text
    120124
     125    def getDbMemory(self, dbType):
     126        if dbType == 'localdatabase' or dbType == 'localdatabase_test':
     127            return self.scratchDbLeaf.find('memory').text
     128        else:
     129            print "memory is not a valid option for non-scratch databases..."
     130            raise
     131
    121132    def setScratchDbLeaf(self, myhost):
    122133        root = self.settingsDoc.getroot()
    … …  
    128139        for loaderhost in localdb.findall('loader'):
    129140          name = loaderhost.get('mach')
    130           print "name, myhost: ", name, myhost
     141          # print "name, myhost: ", name, myhost
    131142          if name == myhost:
    132143              self.scratchDbLeaf = loaderhost
    … …  
    135146        for loaderhost in localdb.findall('loader'):
    136147          name = loaderhost.get('mach')
    137           print "name, myhost: ", name, myhost
     148          # print "name, myhost: ", name, myhost
    138149          if name == 'generic':
    139150              self.scratchDbLeaf = loaderhost

  • trunk/ippToPsps/jython/datastore.py

    r35097 r35417  
    4747                    --list " + tempFile.name
    4848
     49        print "dsreg cmd: ", command
    4950        p = Popen(command, shell=True, stdout=PIPE)
    5051        p.wait()

  • trunk/ippToPsps/jython/detectionbatch.py

    r35171 r35417  
    212212        )"
    213213
    214         print "frame meta sql: ", sql
     214        # print "frame meta sql: ", sql
    215215
    216216        self.scratchDb.execute(sql)
    … …  
    607607         WHERE b.imageID = " + str(imageID) + \
    608608         " INTO OUTFILE '" + dumpFile + "'"
    609         print "sql: ", sql
    610         self.scratchDb.execute(sql)
     609
     610        try: self.scratchDb.execute(sql)
     611        except:
     612            self.logger.info("failed to select data for detectionCalib")
     613            self.logger.infoPair("sql: ", sql)
     614            raise
    611615
    612616        sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName
    613         print "sql: ", sql
    614         self.scratchDb.execute(sql)
     617        try: self.scratchDb.execute(sql)
     618        except:
     619            self.logger.info("failed to load data from infile for detectionCalib")
     620            self.logger.infoPair("sql: ", sql)
     621            raise
    615622       
    616623    '''
    … …  
    655662
    656663        # load corresponding header into memory
    657         header = self.fits.findAndReadHeader(extname)
     664        header = self.fits.findAndReadHeader(extname, self.config.test)
    658665        if not header:
    659             self.logger.errorPair("No header found for chip", chipname)
     666            # XXX There is no way to check if we SHOULD find a given
     667            # chip.  We have to accept the absence of the header as
     668            # information that the chip is not available.  This
     669            # message is too verbose, only print in test mode
     670            if self.config.test: self.logger.errorPair("No header found for chip", chipname)
    660671            return False
    661672       
    … …  
    670681        # store these for later
    671682        self.imageIDs[chipname] = header['IMAGEID']
     683        self.sourceIDs[chipname] = header['SOURCEID']
    672684           
    673685        # populate ImageMeta
    … …  
    691703        # dictionary objects to hold imageIDs for later
    692704        self.imageIDs = {}
     705        self.sourceIDs = {}
    693706
    694707        # loop through all OTAs and populate ImageMeta extensions
    … …  
    773786        # this is a bit crude: if the chip is not present, this test will fail and the chip
    774787        # will be (correctly) skipped.  would be better to carry that information explicitly ("chip is missing")
    775         if not self.scratchDb.astrometricSolutionOK(chipname):
    776             self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
     788
     789        # XXX keep this in or not?
     790        # if not self.scratchDb.astrometricSolutionOK(chipname):
     791        #     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
     792        #     return False
     793
     794        # does this chip exist in the DVO image table (if NOT, then skip it)
     795        if not self.scratchDb.haveThisChip(self.imageIDs[chipname], self.sourceIDs[chipname]):
     796            self.logger.info("| %5s |            ------------- Chip not in DVO : rejecting ------------                    |" % chipname)
    777797            return False
     798
    778799        self.logger.info("populate stuff ");
    779800        # populate remainder of tables
    … …  
    803824        self.totalNullObjID    = self.totalNullObjID + results['NULLOBJID']
    804825        self.totalDetections   = self.totalDetections + rowCount
     826
     827        fractionKept = float(rowCount) / float(results['ORIGINALTOTAL'])
     828        # print "kept ", str(fractionKept), " detections from ", chipname
     829
     830        if fractionKept < 0.8:
     831            # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname
     832            self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname)
     833            self.skipBatch = True
     834            return False;
     835
    805836        #self.logger.info("updated totals")
    806837        # check we have something in this Detection table TODO add this to table above
    807838        if rowCount < 1:
    808             self.logger.infoPair("Skipping empty table for chipname", chipname)
     839            self.logger.infoPair("Empty table for chip, fail on exposure", chipname)
     840            self.skipBatch = True
    809841            return False;
    810842
    … …  
    849881
    850882        # loop through all OTAs again to update with DVO IDs
     883        self.skipBatch = False
    851884        self.validChips = []   
    852885        self.tablesToExport = []   
    … …  
    872905
    873906                if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1
    874 
    875         if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
     907                if self.skipBatch:
     908                    self.logger.error("fatal problem for exposure, skipping")
     909                    return False
     910
     911        if "Chip" in self.imageIDs:
     912            if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
     913            if self.skipBatch:
     914                self.logger.error("fatal problem for exposure, skipping")
     915                return False
    876916
    877917        # print totals
    … …  
    886926        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
    887927
    888         # if we only have one table export, i.e. FrameMeta, then get out of here
     928        # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort)
    889929        if len(self.tablesToExport) == 1:
    890            
     930            self.skipBatch = True
    891931            self.logger.error("No tables to export")
    892932            return False

  • trunk/ippToPsps/jython/dvo.py

    r35223 r35417  
    193193            p = Popen(cmd, shell=True, stdout=PIPE)
    194194            p.wait()
     195            if p.returncode != 0:
     196                self.logger.errorPair("dvopsps failed on SkyTable.fits for ", self.skychunk.dvoLocation)
     197                raise
    195198
    196199        self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable)
    … …  
    591594    def nativeIngestDetections(self, boxId, boxDim):
    592595
    593         # XXX put the chunk below in a separate method
    594         # blow away existing dvoDetection table
    595 
    596596        # clear the 'ingested' field for all boxes owned by this host
    597597        self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost)
    … …  
    607607        ##     raise
    608608
    609         # drop detections table
    610         self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable)
    611         self.scratchDb.dropTable(self.scratchDb.dvoDetectionTable)
    612        
    613         # create detections table
    614         self.logger.infoPair("Creating table", self.scratchDb.dvoDetectionTable)
    615         sql = "CREATE TABLE " + self.scratchDb.dvoDetectionTable + " LIKE dvoDetection"
    616         try: self.scratchDb.execute(sql)
    617         except:
    618             self.logger.errorPair("Unable to create table", self.scratchDb.dvoDetectionTable)
     609        # blow away existing dvoDetection table & re-crate
     610        if not self.scratchDb.resetDvoDetectionTable():
     611            self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
    619612            return False
    620         self.scratchDb.changeEngineToInnoDB(self.scratchDb.dvoDetectionTable)
    621        
    622         # add fileID column
    623         sql = "ALTER TABLE " + self.scratchDb.dvoDetectionTable + " ADD fileID INT NOT NULL"
    624         try: self.scratchDb.execute(sql)
    625         except:
    626             self.logger.errorPair("Unable to add fileID column to ", self.scratchDb.dvoDetectionTable)
    627             return False
    628 
    629         # add an index to the fileID column
    630         self.scratchDb.createIndex(self.scratchDb.dvoDetectionTable, "fileID")
    631613
    632614        # make sure we have an up-to-date Images table
    … …  
    665647        p = Popen(cmd, shell=True, stdout=PIPE)
    666648        p.wait()
     649        if p.returncode != 0:
     650            self.logger.errorPair("dvopsps failed on ", self.scratchDb.dvoDetectionTable)
     651            raise
    667652
    668653        ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding?
    … …  
    670655        self.ippToPspsDb.setIngestedBox(boxId, self.scratchDb.dbHost)
    671656
    672         # add fileID column
     657        # run 'ANALYZE' to get indexes up-to-snuff
    673658        sql = "ANALYZE TABLE " + self.scratchDb.dvoDetectionTable
    674659        try: self.scratchDb.execute(sql)

  • trunk/ippToPsps/jython/dvoobjects.py

    r35223 r35417  
    6767
    6868        if self.skychunk.parallel:
     69            if self.scratchDb.dbHost == "localhost":
     70                self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb")
     71                raise
     72               
    6973            cmd += " -parallel"
    7074
    … …  
    7276        p = Popen(cmd, shell=True, stdout=PIPE)
    7377        p.wait()
     78        if p.returncode != 0:
     79            self.logger.errorPair("dvopsps failed on ", region)
     80            raise
    7481
    7582        # update lists after attempted sync

  • trunk/ippToPsps/jython/fits.py

    r35097 r35417  
    8787
    8888    2880 is the magic number that the FITS format uses to break up data: extensions always begin on multiples of 2880 bytes
     89
     90    Note that we may not be able to distinguish the case where a header is expected from one where the absence of a header
     91    just tells us that the header is not available.  In some cases, we may not want to emit the error message ("not found")
     92
     93    XXX WOW! this block is reading the entire file to get the header.
     94    major inefficiency...  I thought the FITS interactions all went
     95    through STILTS
    8996    '''
    90     def findAndReadHeader(self, name):
     97    def findAndReadHeader(self, name, VERBOSE):
    9198
    9299        found = False
    … …  
    114121
    115122            self.file.seek(origIndex, 0)
    116             self.logger.errorPair("Could not read header", name)
     123            if VERBOSE: self.logger.errorPair("Could not read header", name)
    117124            return
    118125        else:

  • trunk/ippToPsps/jython/gpc1db.py

    r35185 r35417  
    2222    def __init__(self, logger, config):
    2323        # define database type
    24         if (config.test):
     24        if (config.gpc1test):
    2525            dbType = "gpc1database_test"
    2626        else:
    … …  
    104104        try:
    105105            # XXX EAM : test output
    106             self.logger.infoPair("sql for dvo items:", sql)
    107106            rs = self.executeQuery(sql)
    108107            while (rs.next()):
    … …  
    111110        except:
    112111            self.logger.exception("Can't query for ids in DVO")
     112            self.logger.infoPair("failed sql:", sql)
    113113            return rows
    114114

  • trunk/ippToPsps/jython/ipptopsps.py

    r35177 r35417  
    5555            raise
    5656
     57        print "check client status"
    5758        self.checkClientStatus()
    5859
    … …  
    142143    def checkClientStatus(self):
    143144
     145        print "update client"
    144146        self.ippToPspsDb.updateClient(self.config.programName, self.HOST, self.PID)
    145147
     148        print "check killed"
    146149        if self.ippToPspsDb.isKilled(self.HOST, self.PID):
    147150            self.exitProgram("killed via Db")
    … …  
    149152        # this loop pauses the process if we have no skychunk or we have set it to pause
    150153        firstTimeIn = True
     154        print "refresh skychunk"
    151155        while not self.refreshSkychunk() or self.ippToPspsDb.isPaused(self.HOST, self.PID):
    152156

  • trunk/ippToPsps/jython/loader.py

    r35226 r35417  
    3636
    3737        if self.skychunk.parallel:
    38             print "PARALLEL dvo"
     38            self.logger.info("using PARALLEL DVO")
    3939        else:
    40             print "SERIAL dvo"
     40            self.logger.info("using SERIAL DVO")
    4141
    4242        # connect to the gpc1 database
    … …  
    198198                            # insertion
    199199                            if not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
    200                                 self.dvoDetections.nativeIngestDetections(boxId, boxDim)
     200                                if not self.dvoDetections.nativeIngestDetections(boxId, boxDim):
     201                                    self.logger.error("Unable to ingest detections with nativeIngest")
     202                                    return False
    201203
    202204                            useFullTables = 1

  • trunk/ippToPsps/jython/mysql.py

    r35222 r35417  
    4646            self.logger.errorPair("Unable to connect to", connectionStr)
    4747            self.connected = False
     48            raise
     49
     50        # determine the mysql major version number:
     51        sql = "SELECT @@version"
     52
     53        try:
     54            rs = self.executeQuery(sql)
     55            rs.first()
     56            fullVersion = rs.getString(1)
     57            majVersion = fullVersion[0:3]
     58            self.version = float(majVersion)
     59           
     60        except:
     61            print "failed to get version number"
    4862            raise
    4963

  • trunk/ippToPsps/jython/objectbatch.py

    r35223 r35417  
    372372        self.logger.infoPair("Forcing uniqueness on", "objID in Object table")
    373373        rowCountBefore = self.scratchDb.getRowCount("Object")
     374
     375        # XXX EAM : note that in mysql versions later than 5.1, this fails
     376        # unless the following is called first:
     377        # set session old_alter_table=1
     378        # follow the command with
     379        # set session old_alter_table=0
     380        # OF COURSE, this fails for mysql version < 5.5...
     381        if self.scratchDb.version > 5.1:
     382            self.scratchDb.execute("set session old_alter_table=1")
     383           
    374384        self.scratchDb.execute("ALTER IGNORE TABLE Object ADD UNIQUE INDEX(objID)")
     385        if self.scratchDb.version > 5.1:
     386            self.scratchDb.execute("set session old_alter_table=0")
     387
    375388        rowCountAfter = self.scratchDb.getRowCount("Object")
    376389        self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore))

  • trunk/ippToPsps/jython/pslogger.py

    r35097 r35417  
    1313   def setup(self, programName, basePath, skychunkName, host, pid, stdout=1, sendToFile=0):
    1414
    15        formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S')
     15       formatter = logging.Formatter('%(asctime)s.%(msecs)03d | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S')
    1616       self.setLevel(logging.INFO)
    1717

  • trunk/ippToPsps/jython/queue.py

    r35226 r35417  
    2929        super(Queue, self).__init__(argv)
    3030
    31         print "inited ipptopsps"
     31        # print "inited ipptopsps"
    3232
    3333        # create various objects
    3434        self.gpc1Db = Gpc1Db(self.logger, self.config)
    35         print "connect to gpc1"
     35        # print "connect to gpc1"
    3636
    3737        self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)
    3838
    39         print "made datastore"
     39        # print "made datastore"
    4040
    4141        # connect to scratch database
    4242        self.scratchDb = ScratchDb(self.logger, self.config)
    4343
    44         print "connect to scratchdb"
     44        # print "connect to scratchdb"
    4545
    4646        try:
    … …  
    5050            raise
    5151
    52         print "got objects"
     52        # print "got objects"
    5353
    5454        self.skychunk.printAll()

  • trunk/ippToPsps/jython/scratchdb.py

    r35226 r35417  
    2929            dbType = "localdatabase"
    3030
    31         print "trying to connect: ", dbType
     31        # print "trying to connect: ", dbType
    3232
    3333        dbName = config.getDbName(dbType)
    … …  
    3838        self.dvoSkyTable = "dvoSkyTable"
    3939        self.dvoPhotcodesTable = "dvoPhotcodes"
     40        self.memoryTableSize = config.getDbMemory(dbType)
    4041
    4142    '''
    … …  
    368369
    369370    '''
     371    Checks whether the given chipID is in the DVO Image table
     372    '''
     373    def haveThisChip(self, imageID, sourceID):
     374
     375        sql = "SELECT EXTERN_ID FROM " \
     376            + self.dvoImagesTable + \
     377            " WHERE EXTERN_ID = " + str(imageID) + \
     378            " AND SOURCE_ID = " + str(sourceID)
     379
     380        try:
     381            rs = self.executeQuery(sql)
     382           
     383            if not rs:
     384                print "missing result set for imageID query"
     385                raise
     386
     387            # if no returned rows, return an error
     388            if not rs.next():
     389                print "DVO lacks imageID ", imageID
     390                print "sql: ", sql
     391                return False
     392
     393            if rs.getInt(1) == int(imageID):
     394                # print "DVO has imageID ", imageID
     395                return True
     396
     397            else:
     398                print "DVO has wrong imageID? ", imageID
     399                print "result was ", rs.getInt(1)
     400                print "sql: ", sql
     401                raise
     402
     403        except:
     404            print "DVO imageID query failed ", imageID
     405            print "sql: ", sql
     406            raise
     407
     408        # how can I reach this?
     409        return True
     410
     411    '''
    370412    Creates a table for for ID matching
    371413    '''
    … …  
    409451               objID BIGINT, \
    410452               flags INT, \
    411                zp REAL, \
    412                zpErr REAL, \
    413                airMass REAL, \
    414                expTime REAL, \
    415                ra FLOAT, \
    416                dec_ FLOAT, \
    417                raErr REAL, \
    418                decErr REAL, \
     453               zp FLOAT, \
     454               zpErr FLOAT, \
     455               airMass FLOAT, \
     456               expTime FLOAT, \
     457               ra DOUBLE, \
     458               dec_ DOUBLE, \
     459               raErr FLOAT, \
     460               decErr FLOAT, \
    419461               PRIMARY KEY (imageID, ippDetectID), \
    420462               KEY (objID, detectID) \
    … …  
    463505    Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this...
    464506    '''
     507    def resetDvoDetectionTable(self):
     508
     509       self.logger.infoPair("Dropping table", self.dvoDetectionTable)
     510       self.dropTable(self.dvoDetectionTable)
     511
     512       # we need a big enough memory table for the incoming detections
     513       print "requested memory in gigs: ", self.memoryTableSize
     514       tableSize = int(self.memoryTableSize) * 1024 * 1024 * 1024
     515       sql = "set @@max_heap_table_size = " + str(tableSize)
     516       print "setting max heap size: ", sql
     517       try: self.execute(sql)
     518       except:
     519           self.logger.errorPair("Unable to set max MEMORY table size for ", self.dvoDetectionTable)
     520           return False
     521
     522       # create detections table
     523       self.logger.infoPair("Creating table", self.dvoDetectionTable)
     524       sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection"
     525       try: self.execute(sql)
     526       except:
     527           self.logger.errorPair("Unable to create table", self.dvoDetectionTable)
     528           return False
     529
     530       # XXX changing the dvoDetectionFull table to MEMORY
     531       # self.changeEngineToInnoDB(self.dvoDetectionTable)
     532
     533       # set engine to memory
     534       sql = "ALTER TABLE " + self.dvoDetectionTable + " ENGINE = MEMORY"
     535       try: self.execute(sql)
     536       except:
     537           self.logger.errorPair("Unable to set engine to MEMORY for ", self.dvoDetectionTable)
     538           return False
     539
     540       return True
     541
     542    '''
     543    Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this...
     544    '''
    465545    def resetAllDvoTables(self):
    466546
    … …  
    485565       self.changeEngineToInnoDB(self.dvoDoneTable)
    486566
    487        # create detections table
    488        self.logger.infoPair("Creating table", self.dvoDetectionTable)
    489        sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection"
    490        try: self.execute(sql)
    491        except:
    492            self.logger.errorPair("Unable to create table", self.dvoDetectionTable)
     567       # blow away existing dvoDetection table & re-crate
     568       if not self.resetDvoDetectionTable():
     569           self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
    493570           return False
    494        self.changeEngineToInnoDB(self.dvoDetectionTable)
    495 
    496        # add fileID column
    497        sql = "ALTER TABLE " + self.dvoDetectionTable + " ADD fileID INT NOT NULL"
    498        try: self.execute(sql)
    499        except:
    500            self.logger.errorPair("Unable to add fileID column to ", self.dvoDetectionTable)
    501            return False
    502 
    503        # add an index to the fileID column
    504        self.createIndex(self.dvoDetectionTable, "fileID")
    505 
    506        # XXX : this should be based on choice of useStilts (which is set in dvo.py and must be moved)
    507        if (False):
    508            # now add a delete cascading foreign key constraint on fileID
    509            sql = "ALTER TABLE dvoDetectionFull \
    510               ADD CONSTRAINT fk_fileID \
    511               FOREIGN KEY (fileID) \
    512               REFERENCES dvoDone(id) \
    513               ON DELETE CASCADE"
    514            try: self.execute(sql)
    515            except:
    516                self.logger.errorPair("Unable to create foreign key on", self.dvoDoneTable)
    517                return False
    518571
    519572       return True

  • trunk/ippToPsps/jython/skychunk.py

    r35179 r35417  
    2828        if self.name == "all": self.rotateSkychunks = True
    2929
    30         print "name: ", self.name
    31         print "new: ", self.createNewSkychunk
    32         print "all: ", self.rotateSkychunks
     30        # print "name: ", self.name
     31        # print "new: ", self.createNewSkychunk
     32        # print "all: ", self.rotateSkychunks
    3333
    3434        self.logger = logger

  • trunk/ippToPsps/jython/stackbatch.py

    r35222 r35417  
    543543        # response = raw_input("updated dvo ")
    544544       
     545        # XXX EAM : note that in mysql versions later than 5.1, this fails
     546        # unless the following is called first:
     547        # set session old_alter_table=1
     548        # follow the command with
     549        # set session old_alter_table=0
     550        # OF COURSE, this fails for mysql version < 5.5...
     551        if self.scratchDb.version > 5.1:
     552            self.scratchDb.execute("set session old_alter_table=1")
     553
    545554        sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)"
    546555       
     556        if self.scratchDb.version > 5.1:
     557            self.scratchDb.execute("set session old_alter_table=0")
     558
    547559        self.scratchDb.execute(sql)
    548560        # response = raw_input("add primary key? ")
    … …  
    806818         ON (a.stackDetectID = b.detectID) where b.imageID = " + str(imageID) + \
    807819         " INTO OUTFILE '" + dumpFile + "'"
    808         print "sql: ", sql
    809         self.scratchDb.execute(sql)
     820
     821        try: self.scratchDb.execute(sql)
     822        except:
     823            self.logger.info("failed to select data for stackDetectionCalib")
     824            self.logger.infoPair("sql: ", sql)
     825            raise
    810826
    811827        sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName
    812         print "sql: ", sql
    813         self.scratchDb.execute(sql)
     828        try: self.scratchDb.execute(sql)
     829        except:
     830            self.logger.info("failed to load data from infile for stackDetectionCalib")
     831            self.logger.infoPair("sql: ", sql)
     832            raise
    814833
    815834        ## XXX write this with the select/insert

  • trunk/ippToPsps/test/fulltest.sh

    r35225 r35417  
    11#!/bin/csh -f
    2 
    3 set stackqueuename = 0
    4 if ($#argv > 0) then
    5   set stackqueuename = $1
    6   echo "stackqueue: $stackqueuename"
    7 endif
    8 
    9 set objectqueuename = 0
    10 if ($#argv > 0) then
    11   set stackqueuename = $1
    12   echo "stackqueue: $stackqueuename"
    13 endif
    142
    153# a bit crude : get the datadir path by interpollation of the PATH element for psconfig
    … …  
    2412# before running this test, you must have a mysql user:
    2513# mkgpc1.sh user localhost eam eam
    26 
    2714set mkgpc1       = 1
    2815set initdb       = 1
    2916set initscratch  = 1
     17set initbatch    = 1
    3018set camqueue     = 1
    31 set initbatch    = 1
    3219set cambatch     = 1
    3320set stackqueue   = 1
    … …  
    3623set objectbatch  = 1
    3724
     25set args    = ""
     26
     27while ("$1" != "")
     28 switch ("$1")
     29  case -mkgpc1
     30    set mkgpc1 = 0
     31    breaksw;
     32  case -initdb
     33    set initdb = 0
     34    breaksw;
     35  case -initscratch
     36    set initscratch = 0
     37    breaksw;
     38  case -initbatch
     39    set initbatch = 0
     40    breaksw;
     41  case -camera
     42    set camqueue = 0
     43    set cambatch = 0
     44    breaksw;
     45  case -stack
     46    set stackqueue = 0
     47    set stackbatch = 0
     48    breaksw;
     49  case -object
     50    set objectqueue = 0
     51    set objectbatch = 0
     52    breaksw;
     53  default:
     54    set args=($args $1);
     55    breaksw;
     56  endsw
     57  shift
     58end
     59
     60set stackqueuename = 0
     61set objectqueuename = 0
     62
    3863set OUTDIR = "testdata"
    3964set queuename = "test"
    4065
    4166if ($mkgpc1) then
     67  echo ""
     68  echo " ---- mkgpc1data.dvo : creating the dvo & gpc1 database entries ----"
    4269  mkgpc1data.dvo
    4370endif
    … …  
    4572# create the ipptopsps database
    4673if ($initdb) then
     74  echo ""
     75  echo " ---- init ipptopsps_test ----"
    4776
    4877  # does the database already exist?
    … …  
    5584# re-create the ipptopsps database
    5685if ($initscratch) then
     86  echo ""
     87  echo " ---- init ipptopsps_test_scratch ----"
    5788
    5889  # create the database (ignore failure if it already exists)
    … …  
    6293endif
    6394
     95#  `name`
     96#  `datastore_product`
     97#  `datastore_type`
     98#  `datastore_publish`
     99#  `dvo_label`
     100#  `dvo_location`
     101#  `min_ra`
     102#  `max_ra`
     103#  `min_dec`
     104#  `max_dec`
     105#  `box_size`
     106#  `base_path`
     107#  `data_release`
     108#  `delete_local`
     109#  `delete_datastore`
     110#  `delete_dxlayer`
     111#  `epoch`
     112#  `survey`
     113#  `psps_survey`
     114#  `queue_P2`
     115#  `queue_ST`
     116#  `queue_OB`
     117#  `active`
     118#  `parallel`
     119
     120# this is poor : we need to queue something in order to generate an init batch
     121# this is because a skychunk carries the info about the datastore used to publish
     122# the batch
     123if ($initbatch) then
     124  echo ""
     125  echo " ---- make skychunk for IN batches ----"
     126
     127  ippjython queue.py -test edit <<EOF
     128initbatch
     129test_ds
     130IPP_PSPS
     1310
     132none
     133none
     1340
     1350
     1360
     1370
     1382
     139none
     1400
     1410
     1421
     1431
     1442012-12-10 00:00:00.0
     1453PI
     1463PI
     1470
     1480
     1490
     1501
     1510
     152y
     153y
     154EOF
     155
     156  echo ""
     157  echo " ---- create init batch ----"
     158
     159  rm -rf $OUTDIR/IN
     160  ippjython loader.py -test initbatch init
     161endif
     162
    64163# this code is sensitive to a previously existing database?
    65164if ($camqueue) then
     165  echo ""
     166  echo " ---- queue P2 batches ----"
     167
    66168  ippjython queue.py -test edit <<EOF
    67169$queuename
    … …  
    94196endif
    95197
    96 if ($initbatch) then
    97   rm -rf $OUTDIR/IN
    98   ippjython loader.py -test $queuename init
    99 endif
    100 
    101198if ($cambatch) then
     199  echo ""
     200  echo " ---- load P2 batches ----"
     201
    102202  rm -rf $OUTDIR/P2
    103203  ippjython loader.py -test $queuename once
    … …  
    106206# this code is sensitive to a previously existing database?
    107207if ($stackqueue) then
     208  echo ""
     209  echo " ---- queue ST batches ----"
     210
    108211  if ($stackqueuename == 0) then
    109212    set stackqueuename = $queuename\_stk
    … …  
    140243
    141244if ($stackbatch) then
     245  echo ""
     246  echo " ---- load ST batches ----"
     247
    142248  if ($stackqueuename == 0) then
    143249    set stackqueuename = $queuename\_stk
    … …  
    149255# this code is sensitive to a previously existing database?
    150256if ($objectqueue) then
     257  echo ""
     258  echo " ---- queue OB batches ----"
     259
    151260  if ($objectqueuename == 0) then
    152261    set objectqueuename = $queuename\_obj
    … …  
    183292
    184293if ($objectbatch) then
     294  echo ""
     295  echo " ---- load OB batches ----"
     296
    185297  if ($objectqueuename == 0) then
    186298    set objectqueuename = $queuename\_obj

  • trunk/ippToPsps/test/mkgpc1data.dvo

    r35188 r35417  
    196196    # PCA2X0Y2
    197197
     198    if ($i == 3)
     199      echo "NOTE: not adding image $i to DVO : this simulates images with poor astrometry"
     200      continue
     201    end
     202
    198203    echo addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
    199204    exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
Note: See TracChangeset for help on using the changeset viewer.