IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Mar 5, 2013, 2:23:09 PM (13 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20130207/ippToPsps
Files:
4 deleted
36 edited
6 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20130207/ippToPsps

  • branches/eam_branches/ipp-20130207/ippToPsps/Makefile.am

    r35097 r35235  
    11# SUBDIRS = config jars jython src
    2 SUBDIRS = config jars jython
     2SUBDIRS = config jars jython perl
    33
    44CLEANFILES = *.pyc *~ core core.*

  • branches/eam_branches/ipp-20130207/ippToPsps/config/ippToPspsDbSchema.sql

    r35097 r35235  
    4040  `deleted_datastore` tinyint(4) default '0',
    4141  `deleted_dxlayer` tinyint(4) default '0',
     42  `block_delete_local` tinyint(4) default '0',
     43  `block_delete_datastore` tinyint(4) default '0',
     44  `block_delete_dxlayer` tinyint(4) default '0',
    4245  `stage_id` bigint(20) default NULL,
    4346  `total_detections` bigint(20) default NULL,
    … …  
    6265  `dec_center` float default NULL,
    6366  `box_side` float default NULL,
    64   `ingested` tinyint(1) default '0',
     67  `ingested` varchar(64) default 'none',
    6568  PRIMARY KEY  (`id`),
    6669  KEY `fk_skychunk` (`skychunk`),

  • branches/eam_branches/ipp-20130207/ippToPsps/config/settings.xml

    r35097 r35235  
    1010  <!-- local scratch Db section (REAL version) -->
    1111  <localdatabase>
    12     <name>ipptopsps_scratch</name>
    13     <host>ipp005</host>
    14     <user>dvo</user>
    15     <password>dvo</password>
     12    <loader mach="ipp006">
     13      <name>ipptopsps_scratch</name>
     14      <host>ipp006</host>
     15      <user>dvo</user>
     16      <password>dvo</password>
     17    </loader>
     18    <loader mach="ipp007">
     19      <name>ipptopsps_scratch</name>
     20      <host>ipp007</host>
     21      <user>dvo</user>
     22      <password>dvo</password>
     23    </loader>
     24    <loader mach="ipp008">
     25      <name>ipptopsps_scratch</name>
     26      <host>ipp008</host>
     27      <user>dvo</user>
     28      <password>dvo</password>
     29    </loader>
     30    <loader mach="ipp009">
     31      <name>ipptopsps_scratch</name>
     32      <host>ipp009</host>
     33      <user>dvo</user>
     34      <password>dvo</password>
     35    </loader>
     36    <loader mach="ipp010">
     37      <name>ipptopsps_scratch</name>
     38      <host>ipp010</host>
     39      <user>dvo</user>
     40      <password>dvo</password>
     41    </loader>
     42    <loader mach="generic">
     43      <name>ipptopsps_scratch</name>
     44      <host>localhost</host>
     45      <user>dvo</user>
     46      <password>dvo</password>
     47    </loader>
    1648  </localdatabase>
    1749
    … …  
    2759  <ipptopspsdatabase>
    2860    <name>ipptopsps</name>
    29     <host>localhost</host>
     61    <host>ipp006</host>
    3062    <user>dvo</user>
    3163    <password>dvo</password>
    … …  
    3466  <!-- local scratch Db section (TEST VERSION) -->
    3567  <localdatabase_test>
    36     <name>ipptopsps_test_scratch</name>
    37     <host>localhost</host>
    38     <user>dvo</user>
    39     <password>dvo</password>
     68    <loader mach="ipp006">
     69      <name>ipptopsps_test_scratch</name>
     70      <host>ipp006</host>
     71      <user>dvo</user>
     72      <password>dvo</password>
     73    </loader>
     74    <loader mach="pikake">
     75      <name>ipptopsps_test_scratch</name>
     76      <host>pikake</host>
     77      <user>dvo</user>
     78      <password>dvo</password>
     79    </loader>
     80    <loader mach="generic">
     81      <name>ipptopsps_test_scratch</name>
     82      <host>localhost</host>
     83      <user>dvo</user>
     84      <password>dvo</password>
     85    </loader>
    4086  </localdatabase_test>
    4187
    42   <!-- GPC1 Db section (TEST VERSION) -->
     88  <!-- GPC1 Db section (REAL VERSION) -->
     89  <gpc1database_test>
     90    <name>gpc1</name>
     91    <host>ippdb01</host>
     92    <user>ipp</user>
     93    <password>ipp</password>
     94  </gpc1database_test>
     95
     96  <!-- GPC1 Db section (TEST VERSION)
    4397  <gpc1database_test>
    4498    <name>gpc1test</name>
    … …  
    46100    <user>dvo</user>
    47101    <password>dvo</password>
    48   </gpc1database_test>
     102  </gpc1database_test> -->
    49103
    50104  <!-- ippToPsps Db section (TEST VERSION)  -->
    … …  
    59113  <czardatabase>
    60114    <name>czardb</name>
    61     <host>none</host>
     115    <host>ippdb01</host>
    62116    <user>ipp</user>
    63117    <password>ipp</password>

  • branches/eam_branches/ipp-20130207/ippToPsps/config/tables.IN.vot

    r35097 r35235  
    578578      </DATA>
    579579    </TABLE>
    580     <TABLE name="SkyCell">
    581       <DESCRIPTION>VOTable description of PSPS table SkyCell</DESCRIPTION>
    582       <PARAM arraysize="1" datatype="char" ucd="meta.bib.author" name="Author" value="PSPS"></PARAM>
    583       <FIELD name="skyCellID" arraysize="1" datatype="int">
    584         <DESCRIPTION>indentifier of the sky cell</DESCRIPTION>
    585       </FIELD>
    586       <FIELD name="projectionCellID" arraysize="1" datatype="int">
    587         <DESCRIPTION>indentifier of the projection cell</DESCRIPTION>
    588       </FIELD>
    589       <FIELD name="partitionKey" arraysize="1" datatype="long">
    590         <DESCRIPTION>fgetPANobjID(skyCell_center_ra,  skyCell_center_dec, zHeight)</DESCRIPTION>
    591       </FIELD>
    592       <FIELD name="regionID" arraysize="1" datatype="long">
    593         <DESCRIPTION>region identifier in the Region table</DESCRIPTION>
    594       </FIELD>
    595       <DATA>
    596         <TABLEDATA>
    597           <TR><TD>1</TD><TD>1</TD><TD>-999</TD><TD>1</TD></TR>
    598           <TR><TD>2</TD><TD>2</TD><TD>-999</TD><TD>1</TD></TR>
    599           <TR><TD>3</TD><TD>3</TD><TD>-999</TD><TD>1</TD></TR>
    600           <TR><TD>4</TD><TD>4</TD><TD>-999</TD><TD>1</TD></TR>
    601           <TR><TD>5</TD><TD>5</TD><TD>-999</TD><TD>1</TD></TR>
    602           <TR><TD>6</TD><TD>6</TD><TD>-999</TD><TD>1</TD></TR>
    603           <TR><TD>7</TD><TD>7</TD><TD>-999</TD><TD>1</TD></TR>
    604           <TR><TD>8</TD><TD>8</TD><TD>-999</TD><TD>1</TD></TR>
    605           <TR><TD>9</TD><TD>9</TD><TD>-999</TD><TD>1</TD></TR>
    606           <TR><TD>10</TD><TD>10</TD><TD>-999</TD><TD>1</TD></TR>
    607         </TABLEDATA>
    608       </DATA>
    609     </TABLE>
    610     <TABLE name="ProjectionCell">
    611       <DESCRIPTION>VOTable description of PSPS table ProjectionCell</DESCRIPTION>
    612       <PARAM arraysize="1" datatype="char" ucd="meta.bib.author" name="Author" value="PSPS"></PARAM>
    613       <FIELD name="projectionCellID" arraysize="1" datatype="int">
    614         <DESCRIPTION>projection cell unique identifier</DESCRIPTION>
    615       </FIELD>
    616       <FIELD name="partitionKey" arraysize="1" datatype="long">
    617         <DESCRIPTION>fGetPanObjID(crval1,crval2)</DESCRIPTION>
    618       </FIELD>
    619       <FIELD name="primaryCellRegionID" arraysize="1" datatype="long">
    620         <DESCRIPTION>reference to the primary cell region definition in the Region table</DESCRIPTION>
    621       </FIELD>
    622       <FIELD name="projectionCellRegionID" arraysize="1" datatype="long">
    623         <DESCRIPTION>reference to the projectio cell region definition in the Region table)</DESCRIPTION>
    624       </FIELD>
    625       <FIELD name="ctype1" arraysize="100" datatype="char">
    626         <DESCRIPTION>name of astrometric projection in RA</DESCRIPTION>
    627       </FIELD>
    628       <FIELD name="ctype2" arraysize="100" datatype="char">
    629         <DESCRIPTION>name of astrometric projection in DEC</DESCRIPTION>
    630       </FIELD>
    631       <FIELD name="crval1" arraysize="1" datatype="double">
    632         <DESCRIPTION> RA corresponding to reference pixel (unit = deg)</DESCRIPTION>
    633       </FIELD>
    634       <FIELD name="crval2" arraysize="1" datatype="double">
    635         <DESCRIPTION> DEC corresponding to reference pixel (unit = deg)</DESCRIPTION>
    636       </FIELD>
    637       <FIELD name="crpix1" arraysize="1" datatype="double">
    638         <DESCRIPTION>reference pixel value for RA</DESCRIPTION>
    639       </FIELD>
    640       <FIELD name="crpix2" arraysize="1" datatype="double">
    641         <DESCRIPTION>reference pixel value for DEC</DESCRIPTION>
    642       </FIELD>
    643       <FIELD name="pc001001" arraysize="1" datatype="double">
    644         <DESCRIPTION>elements of rotation/Dcale matrix</DESCRIPTION>
    645       </FIELD>
    646       <FIELD name="pc001002" arraysize="1" datatype="double">
    647         <DESCRIPTION>elements of rotation/Dcale matrix</DESCRIPTION>
    648       </FIELD>
    649       <FIELD name="pc002001" arraysize="1" datatype="double">
    650         <DESCRIPTION>elements of rotation/Dcale matrix</DESCRIPTION>
    651       </FIELD>
    652       <FIELD name="pc002002" arraysize="1" datatype="double">
    653         <DESCRIPTION>elements of rotation/Dcale matrix</DESCRIPTION>
    654       </FIELD>
    655       <DATA>
    656         <TABLEDATA>
    657           <TR><TD>1</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    658           <TR><TD>2</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    659           <TR><TD>3</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    660           <TR><TD>4</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    661           <TR><TD>5</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    662           <TR><TD>6</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    663           <TR><TD>7</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    664           <TR><TD>8</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    665           <TR><TD>9</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    666           <TR><TD>10</TD><TD>-999</TD><TD>1</TD><TD>1</TD></TR>
    667         </TABLEDATA>
    668       </DATA>
    669     </TABLE>
    670     <TABLE name="Region">
    671       <DESCRIPTION>VOTable description of PSPS table Region</DESCRIPTION>
    672       <PARAM arraysize="1" datatype="char" ucd="meta.bib.author" name="Author" value="PSPS"></PARAM>
    673       <FIELD name="regionID" arraysize="1" datatype="long">
    674         <DESCRIPTION>Unique Region identifier</DESCRIPTION>
    675       </FIELD>
    676       <FIELD name="id" arraysize="1" datatype="long">
    677         <DESCRIPTION>Custom Identifier</DESCRIPTION>
    678       </FIELD>
    679       <FIELD name="type" arraysize="100" datatype="char">
    680         <DESCRIPTION>Region type</DESCRIPTION>
    681       </FIELD>
    682       <FIELD name="comment" arraysize="100" datatype="char">
    683         <DESCRIPTION>Comment</DESCRIPTION>
    684       </FIELD>
    685       <FIELD name="area" arraysize="1" datatype="double">
    686         <DESCRIPTION>Area covered by the region</DESCRIPTION>
    687       </FIELD>
    688       <FIELD name="regionBinary" arraysize="100" datatype="char">
    689         <DESCRIPTION>Binary object managed by the spatial library</DESCRIPTION>
    690       </FIELD>
    691       <DATA>
    692         <TABLEDATA>
    693           <TR><TD>1</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    694           <TR><TD>2</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    695           <TR><TD>3</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    696           <TR><TD>4</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    697           <TR><TD>5</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    698           <TR><TD>6</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    699           <TR><TD>7</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    700           <TR><TD>8</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    701           <TR><TD>9</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    702           <TR><TD>10</TD><TD>-999</TD><TD>a type</TD><TD>a comment</TD><TD>-999.0</TD><TD> </TD></TR>
    703         </TABLEDATA>
    704       </DATA>
    705     </TABLE>
    706580    <TABLE name="StackType">
    707581      <DESCRIPTION>VOTable description of PSPS table StackType</DESCRIPTION>
    … …  
    788662          <TR><TD>RADIAL_FLUX</TD><TD>134217728</TD><TD>radial flux measurements calculated</TD></TR>
    789663          <TR><TD>SIZE_SKIPPED</TD><TD>268435456</TD><TD>size could not be determined</TD></TR>
    790           <TR><TD>ON_SPIKE</TD><TD>536870912</TD><TD>peak lands on diffraction spike</TD></TR>
    791           <TR><TD>ON_GHOST</TD><TD>1073741824</TD><TD>peak lands on ghost or glint</TD></TR>
    792           <TR><TD>OFF_CHIP</TD><TD>2147483648</TD><TD>peak lands off edge of chip</TD></TR>
     664          <TR><TD>PEAK_ON_SPIKE</TD><TD>536870912</TD><TD>peak lands on diffraction spike</TD></TR>
     665          <TR><TD>PEAK_ON_GHOST</TD><TD>1073741824</TD><TD>peak lands on ghost or glint</TD></TR>
     666          <TR><TD>PEAK_OFF_CHIP</TD><TD>2147483648</TD><TD>peak lands off edge of chip</TD></TR>
    793667          <TR><TD>DIFF_WITH_SINGLE</TD><TD>4294967296</TD><TD>diff source matched to a single positive detection</TD></TR>
    794668          <TR><TD>DIFF_WITH_DOUBLE</TD><TD>8589934592</TD><TD>diff source matched to positive detections in both images</TD></TR>
    … …  
    868742          <TR><TD>RADIAL_FLUX</TD><TD>134217728</TD><TD>radial flux measurements calculated</TD></TR>
    869743          <TR><TD>SIZE_SKIPPED</TD><TD>268435456</TD><TD>size could not be determined</TD></TR>
    870           <TR><TD>ON_SPIKE</TD><TD>536870912</TD><TD>peak lands on diffraction spike</TD></TR>
    871           <TR><TD>ON_GHOST</TD><TD>1073741824</TD><TD>peak lands on ghost or glint</TD></TR>
    872           <TR><TD>OFF_CHIP</TD><TD>2147483648</TD><TD>peak lands off edge of chip</TD></TR>
     744          <TR><TD>PEAK_ON_SPIKE</TD><TD>536870912</TD><TD>peak lands on diffraction spike</TD></TR>
     745          <TR><TD>PEAK_ON_GHOST</TD><TD>1073741824</TD><TD>peak lands on ghost or glint</TD></TR>
     746          <TR><TD>PEAK_OFF_CHIP</TD><TD>2147483648</TD><TD>peak lands off edge of chip</TD></TR>
    873747          <TR><TD>DIFF_WITH_SINGLE</TD><TD>4294967296</TD><TD>diff source matched to a single positive detection</TD></TR>
    874748          <TR><TD>DIFF_WITH_DOUBLE</TD><TD>8589934592</TD><TD>diff source matched to positive detections in both images</TD></TR>

  • branches/eam_branches/ipp-20130207/ippToPsps/config/tables.OB.vot

    r35097 r35235  
    300300        <DESCRIPTION>Data release</DESCRIPTION>
    301301      </FIELD>
    302       <FIELD name="updated" arraysize="1" datatype="int" unit="unknown">
    303         <DESCRIPTION>MJD of last update</DESCRIPTION>
     302      <FIELD name="batchID" arraysize="1" datatype="int" unit="unknown">
     303        <DESCRIPTION>batch id</DESCRIPTION>
    304304      </FIELD>
    305305      <DATA>

  • branches/eam_branches/ipp-20130207/ippToPsps/config/tables.P2.vot

    r35097 r35235  
    147147        <DESCRIPTION>calibration modification number</DESCRIPTION>
    148148      </FIELD>
     149      <FIELD name="batchID" arraysize="1" datatype="int" unit="unknown">
     150        <DESCRIPTION>batchID</DESCRIPTION>
     151      </FIELD>
    149152      <FIELD name="dataRelease" arraysize="1" datatype="unsignedByte" unit="unknown">
    150153        <DESCRIPTION>Data release</DESCRIPTION>

  • branches/eam_branches/ipp-20130207/ippToPsps/config/tables.ST.vot

    r35097 r35235  
    105105        <DESCRIPTION>calibration modification number</DESCRIPTION>
    106106      </FIELD>
     107      <FIELD name="batchID" arraysize="1" datatype="int" unit="unknown">
     108        <DESCRIPTION>batchID</DESCRIPTION>
     109      </FIELD>
    107110      <FIELD name="dataRelease" arraysize="1" datatype="unsignedByte">
    108111        <DESCRIPTION>Data release</DESCRIPTION>

  • branches/eam_branches/ipp-20130207/ippToPsps/configure.ac

    r35097 r35235  
    4949  jars/Makefile
    5050  jython/Makefile
     51  perl/Makefile
    5152  src/Makefile
    5253  Doxyfile

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/batch.py

    r35097 r35235  
    544544        # for OB, this imports object data from DVO
    545545        if not self.importIppTables():
     546            if self.batchType == "OB":
     547                self.logger.errorPair("skipping this batch", "no dvo tables for this region")
     548                self.ippToPspsDb.updateProcessed(self.batchID, 1)
     549                return True
     550
    546551            self.logger.errorPair("Aborting this batch", "could not import IPP tables")
    547552            self.ippToPspsDb.updateProcessed(self.batchID, -1)

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/config.py

    r35097 r35235  
    3838        ## output file.  we blindly set name == argv[1]
    3939        ## programs which are called without argv[1] will use 'none' in the log file
     40        self.name = None
    4041        if len(sys.argv) >= 2:
    4142            self.name = sys.argv[1]
    … …  
    5758        self.settingsDoc = ElementTree(file=self.settingsPath)
    5859        self.logPath = self.settingsDoc.find("logPath").text
     60        self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text
     61
     62
    5963
    6064        print "config.programName: ", self.programName
    … …  
    6367        print "config.logPath: ", self.logPath
    6468        print "config.test: ", self.test
     69        print "config.czarPlotsPath: ", self.czarPlotsPath
     70
     71        root = self.settingsDoc.getroot()
     72        localdb = root.find('localdatabase')
     73        for loaderhost in localdb.findall('loaderhost'):
     74          name = loaderhost.get('name')
     75          dbname = loaderhost.find('dbname').text
     76          dbhost = loaderhost.find('host').text
     77          print "loaderhost / dbhost : ", name, dbname, dbhost
     78
    6579
    6680    '''
    … …  
    6983    def getLogger(self, host, pid, stdout=1, sendToFile=0):
    7084       
    71         print "get Logger: ", host, pid
    7285        logging.setLoggerClass(PSLogger)
    73         print "done Logger 1"
    7486        self.logger = logging.getLogger(self.programName)
    75         print "done Logger 2"
    7687        if (self.name is None):
    7788            self.logger.setup(self.programName, self.logPath, "none", host, pid, stdout, sendToFile)
    7889        else:
    7990            self.logger.setup(self.programName, self.logPath, self.name, host, pid, stdout, sendToFile)
    80         print "done Logger 3"
    8191
    8292        return self.logger
    … …  
    8595    Various db metadata getters
    8696    '''
    87     def getDbName(self, dbType): return self.settingsDoc.find(dbType +"/name").text
    88     def getDbHost(self, dbType): return self.settingsDoc.find(dbType +"/host").text
    89     def getDbUser(self, dbType): return self.settingsDoc.find(dbType +"/user").text
    90     def getDbPassword(self, dbType): return self.settingsDoc.find(dbType +"/password").text
     97    def getDbName(self, dbType):
     98        if dbType == 'localdatabase' or dbType == 'localdatabase_test':
     99            return self.scratchDbLeaf.find('name').text
     100        else:
     101            return self.settingsDoc.find(dbType +"/name").text
     102
     103    def getDbHost(self, dbType):
     104        if dbType == 'localdatabase' or dbType == 'localdatabase_test':
     105            return self.scratchDbLeaf.find('host').text
     106        else:
     107            return self.settingsDoc.find(dbType +"/host").text
     108
     109    def getDbUser(self, dbType):
     110        if dbType == 'localdatabase' or dbType == 'localdatabase_test':
     111            return self.scratchDbLeaf.find('user').text
     112        else:
     113            return self.settingsDoc.find(dbType +"/user").text
     114
     115    def getDbPassword(self, dbType):
     116        if dbType == 'localdatabase' or dbType == 'localdatabase_test':
     117            return self.scratchDbLeaf.find('password').text
     118        else:
     119            return self.settingsDoc.find(dbType +"/password").text
     120
     121    def setScratchDbLeaf(self, myhost):
     122        root = self.settingsDoc.getroot()
     123        if (self.test):
     124            localdb = root.find('localdatabase_test')
     125        else:
     126            localdb = root.find('localdatabase')
     127
     128        for loaderhost in localdb.findall('loader'):
     129          name = loaderhost.get('mach')
     130          print "name, myhost: ", name, myhost
     131          if name == myhost:
     132              self.scratchDbLeaf = loaderhost
     133              return 1
     134
     135        for loaderhost in localdb.findall('loader'):
     136          name = loaderhost.get('mach')
     137          print "name, myhost: ", name, myhost
     138          if name == 'generic':
     139              self.scratchDbLeaf = loaderhost
     140              return 1
     141
     142        print "Config Error: cannot find valid scratch db matching ", myhost, "or generic"
     143        raise

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/detectionbatch.py

    r35124 r35235  
    215215
    216216        self.scratchDb.execute(sql)
    217 
     217        self.scratchDb.updateAllRows("FrameMeta", "batchID", str(self.batchID))
    218218        self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID))
    219219        self.scratchDb.updateFilterID("FrameMeta", self.filter)

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/dvo.py

    r35097 r35235  
    66import sys
    77import os
     8import math
    89import logging
    910import glob
    … …  
    3334
    3435    '''
    35     def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None):
     36    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
    3637
    3738        # set up logging
    … …  
    4041        self.skychunk = skychunk
    4142        self.ippToPspsDb = ippToPspsDb
     43        self.scratchDb = scratchDb
    4244
    4345        # set up empty lists
    … …  
    4850        self.regionsIngestedButOutOfDate = []
    4951       
    50         # connect to the specified scratch database, if a name is given, otherwise use the first version
    51         try:
    52             self.scratchDb = ScratchDb(self.logger, self.config, '1', scratchDbName)
    53         except: raise
    54 
    5552        # we have (still) 3 modes of ingest from dvo cpt/cps/cpm files
    5653        # 1) dvograbber (useFullTables == FALSE),
    … …  
    211208    def setSkyAreaAsStripe(self, ra=0., width=4.):
    212209
     210        print "this function should not reached now, error"
     211        raise
     212
    213213        self.setSkyArea(ra, ra+width, -91., 91.)
    214214
    … …  
    217217    '''
    218218    def setSkyAreaAsBox(self, ra=180., dec=0., side=181.0):
     219
     220        print "this function should not reached now, error"
     221        raise
    219222
    220223        halfSide = side/2.0
    … …  
    223226    '''
    224227    Determines what DVO files need to be ingested to the defines region
     228    XXX EAM : this function has been substantially reduced / stripped down to work with the parallel dvo
     229    XXX EAM : some additional review of the goals would be useful
    225230    '''
    226231    def setSkyArea(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91):
    … …  
    263268
    264269           # get paths for this region for the file types we are interested in
    265            paths = []
    266            for fileType in self.ingestFileTypes:
    267                paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
     270           # XXX ipptopsps cannot check files -- it does not know where they are
     271           ## paths = []
     272           ## for fileType in self.ingestFileTypes:
     273           ##     paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
    268274
    269275           # check for the existence of all interested file types
    270            skipThisRegion = False
    271            for path in paths:
    272                if not os.path.isfile(path):
    273                    skipThisRegion = True
    274                    break
    275 
    276            if skipThisRegion: continue
     276           # XXX ipptopsps cannot check files -- it does not know where they are
     277           ## skipThisRegion = False
     278           ## for path in paths:
     279           ##     if not os.path.isfile(path):
     280           ##         skipThisRegion = True
     281           ##         break
     282
     283           ## if skipThisRegion: continue
    277284
    278285           # ok, have real, populated files so add this region to the list
    … …  
    281288
    282289           # if we have already imported up-to-date versions of all interested files, then we skip this region
    283            alreadyIngested = True
    284            for path in paths:
    285                if not self.scratchDb.alreadyImportedThisDvoTable(path):
    286                    alreadyIngested = False
    287                    break
    288    
     290           ## alreadyIngested = True
     291           ## for path in paths:
     292           ##     if not self.scratchDb.alreadyImportedThisDvoTable(path):
     293           ##         alreadyIngested = False
     294           ##         break
     295
     296           alreadyIngested = False
    289297           if alreadyIngested:
    290298               self.regionsAlreadyIngested.append(regionPath)
    … …  
    299307
    300308           # check if we have out-of-date versions of any interested files, if so, add region to the purge list
    301            outOfDate = False
    302            for path in paths:
    303                if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path):
    304                    outOfDate = True
    305                    break
    306 
    307            if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
     309           ## outOfDate = False
     310           ## for path in paths:
     311           ##     if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path):
     312           ##         outOfDate = True
     313           ##         break
     314           ##
     315           ## if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
     316
     317           print "using region: ", regionPath
    308318
    309319           self.regionsToIngest.append(regionPath)
    … …  
    389399        self.logger.infoPair("Purging", "%d regions" % len(regions))
    390400
    391         detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
    392         if detectionDeleteCount > 0: self.logger.infoPair("Deleted", "%d detections" % detectionDeleteCount)
     401        self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
     402
     403        ## detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
     404        ## if detectionDeleteCount > 0: self.logger.infoPair("Deleted", "%d detections" % detectionDeleteCount)
    393405
    394406        return True
    … …  
    420432
    421433        size = 0.0
     434        return size
     435
    422436        for region in regions:
    423437             self.logger.infoPair("sizes for region", region)
    424438             # get combined size of all interested files
    425439             for fileType in self.ingestFileTypes:
     440                 print "trying ", fileType
    426441                 size = size + self.getDiskSize(self.skychunk.dvoLocation + "/" + region + "." + fileType)
    427442                 # EAM TEST I/O
    … …  
    574589    includes purging detections outside sky area
    575590    '''
    576     def nativeIngestDetections(self, boxId, raCenter, decCenter, boxSize):
     591    def nativeIngestDetections(self, boxId, boxDim):
    577592
    578593        # XXX put the chunk below in a separate method
    579594        # blow away existing dvoDetection table
    580595
    581         # clear the 'ingested' field for all boxes
    582         self.ippToPspsDb.clearIngestedBoxes()
     596        # clear the 'ingested' field for all boxes owned by this host
     597        self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost)
    583598       
     599        ## print "*******************************************************************************"
     600        ## self.logger.warning("about to delete detection table and re-ingest; is this correct?")
     601        ## response = raw_input("(y/n) ")
     602        ## if response != "y":
     603        ##     raise
     604        ## self.logger.warning("Are you ABSOLUTELY sure you want to do this?")
     605        ## response = raw_input("(y/n) ")
     606        ## if response != "y":
     607        ##     raise
     608
    584609        # drop detections table
    585610        self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable)
    … …  
    608633        self.loadImages()
    609634
    610         # dvopsps -D catdir CATDIR -region .... -dbhost xx -dbname xx -dbuser xx -dbpass xx
    611         halfSize = boxSize / 2.0
    612         # -region raCenter-halfSize raCenter+halfSize decCenter-halfSize decCenter+halfSize
     635        # the box dimensions are the area used to select the items of
     636        # interest from the gpc1 database (camera smfs / skycal cmfs).
     637        # for dvopsps, we need to include a border region large enough
     638        # to grab all detections / objects which are contributed by an
     639        # item with the center in the given box
     640        dR_border = self.skychunk.BORDER / math.cos(math.radians(boxDim['DEC']))
     641        dD_border = self.skychunk.BORDER
     642
     643        minRA  = boxDim['minRA']  - dR_border
     644        maxRA  = boxDim['maxRA']  + dR_border
     645        minDEC = boxDim['minDEC'] - dD_border
     646        maxDEC = boxDim['maxDEC'] + dD_border
    613647
    614648        # TODO path to DVO prog hardcoded temporarily
    … …  
    620654        cmd += " -D CATDIR " + self.skychunk.dvoLocation
    621655        cmd += " -region "
    622         cmd += " " + str(raCenter-halfSize)
    623         cmd += " " + str(raCenter+halfSize)
    624         cmd += " " + str(decCenter-halfSize)
    625         cmd += " " + str(decCenter+halfSize)
     656        cmd += " " + str(minRA)
     657        cmd += " " + str(maxRA)
     658        cmd += " " + str(minDEC)
     659        cmd += " " + str(maxDEC)
    626660
    627661        if self.skychunk.parallel:
    … …  
    632666        p.wait()
    633667
    634         self.ippToPspsDb.setIngestedBox(boxId)
     668        ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding?
     669
     670        self.ippToPspsDb.setIngestedBox(boxId, self.scratchDb.dbHost)
    635671
    636672        # add fileID column

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/dvodetections.py

    r35097 r35235  
    2020    Constructor
    2121    '''
    22     def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName):
     22    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
    2323
    24         super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDbName)
     24        super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb)
    2525
    2626        # declare DVO file types of interest

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/dvoobjects.py

    r35097 r35235  
    2020    Constructor
    2121    '''
    22     def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None):
     22    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
    2323
    24         super(DvoObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDbName)
     24        super(DvoObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb)
    2525
    2626        # declare DVO file types of interest

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/dxlayer.py

    r32934 r35235  
    3030        tempFile = tempfile.NamedTemporaryFile(mode='w+b')
    3131
    32         p = Popen("../perl/deleteFromDXLayer.pl -b " + batch + " -f " + tempFile.name,
     32        p = Popen("deleteFromDXLayer.pl -b " + batch + " -f " + tempFile.name,
    3333                shell=True,
    3434                stdout=PIPE)

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/gpc1db.py

    r35097 r35235  
    6161            sql = "SELECT DISTINCT stage_id, DEGREES(ra), DEGREES(decl) \
    6262                   FROM addRun \
     63                   JOIN addProcessedExp USING (add_id) \
    6364                   JOIN minidvodbRun USING(minidvodb_name) \
    6465                   JOIN minidvodbProcessed USING(minidvodb_id) \
    6566                   JOIN camRun ON (stage_id = cam_id) \
     67                   JOIN camProcessedExp USING(cam_id) \
    6668                   JOIN chipRun USING (chip_id) \
    6769                   JOIN rawExp USING (exp_id) \
    … …  
    7375                   AND minidvodbProcessed.fault = 0 \
    7476                   AND mergedvodbProcessed.fault = 0 \
     77                   AND camProcessedExp.fault = 0 \
     78                   AND addProcessedExp.fault = 0 \
    7579                   AND addRun.stage = '" + stage + "' \
    7680                   AND addRun.state = 'full' \

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/ipptopsps.py

    r35097 r35235  
    4040        self.MINUTES = None
    4141        self.SECONDS = None
     42
     43        self.config.setScratchDbLeaf(self.HOST)
    4244
    4345        self.logger = self.config.getLogger(self.HOST, self.PID, logToStdout, logToFile)

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/ipptopspsdb.py

    r35097 r35235  
    44import sys
    55import os
     6import math
    67import logging
    78
    … …  
    997998
    998999            self.skychunk.boxSize = rs.getDouble(10)
    999             self.skychunk.halfBox = self.skychunk.boxSize/2.0
    1000             self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2)
     1000            # self.skychunk.halfBox = self.skychunk.boxSize/2.0
     1001            # self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2)
    10011002
    10021003            self.skychunk.basePath = rs.getString(11)
    … …  
    10311032    '''
    10321033    Gets id of box with these coords
     1034    XXX EAM : this code implies we can have multiple boxes at the same location for the same
     1035    XXX EAM : skychunk distinguished by having different sizes.  that seems quite bad
    10331036    '''
    10341037    def getBoxId(self, ra, dec):
     1038
     1039        # for the float comparison, need to use a small window
     1040        minRA = ra - 0.01
     1041        maxRA = ra + 0.01
     1042        minDEC = dec - 0.01
     1043        maxDEC = dec + 0.01
    10351044
    10361045        sql = "SELECT id FROM box \
    10371046               WHERE skychunk  = '" + self.skychunk.name + "' \
    1038                AND ra_center = " + str(ra) + " \
    1039                AND dec_center = " + str(dec) + " \
    1040                AND box_side = " + str(self.skychunk.boxSize)
    1041 
     1047               AND ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
     1048               AND dec_center BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
     1049
     1050               # AND box_side = " + str(self.skychunk.boxSize)
     1051
     1052        # print "sql for boxid: ", sql
     1053       
    10421054        id = -1
    10431055        try:
    … …  
    10791091    Inserts new box into box table. If it already exisits, it returns existing box id
    10801092    '''
    1081     def isBoxIngested(self, id):
     1093    def isBoxIngested(self, id, host):
    10821094
    10831095        # set the field 'ingested' to 1 for this box
    … …  
    10881100                print "no matching boxes for ", str(id)
    10891101                raise
    1090             status = rs.getInt(1)
     1102            value = rs.getString(1)
     1103        except:
     1104            print "failed to find box for ", str(id)
     1105            raise
     1106
     1107        if value == host:
     1108            return 1
     1109
     1110        if value == "none":
     1111            return 0
     1112
     1113        if value is None:
     1114            return 0
     1115
     1116        print "box ingested host ", value, " does not match db host ", host
     1117        raise
     1118
     1119    '''
     1120    Inserts new box into box table. If it already exisits, it returns existing box id
     1121    '''
     1122    def setIngestedBox(self, id, host):
     1123
     1124        # set the field 'ingested' to 1 for this box
     1125        sql = "update box set ingested = '" + host + "' where id = " + str(id)
     1126        try:
     1127            self.execute(sql)
    10911128        except:
    10921129            print "failed to set box ", str(id), "as ingested"
    10931130            raise
    10941131
    1095         return status
    1096     '''
    1097     Inserts new box into box table. If it already exisits, it returns existing box id
    1098     '''
    1099     def setIngestedBox(self, id):
     1132    '''
     1133    Clear ingested flag for boxes owned by this host
     1134    '''
     1135    def clearIngestedBoxes(self, host):
    11001136
    11011137        # set the field 'ingested' to 1 for this box
    1102         sql = "update box set ingested = 1 where id = " + str(id)
     1138        sql = "update box set ingested = 'none' where ingested = '" + host + "'"
    11031139        try:
    11041140            self.execute(sql)
    11051141        except:
    1106             print "failed to set box ", str(id), "as ingested"
     1142            print "failed to clear ingested state for boxes owned by host ", host
    11071143            raise
    11081144
    11091145    '''
    1110     Inserts new box into box table. If it already exisits, it returns existing box id
    1111     '''
    1112     def clearIngestedBoxes(self):
    1113 
    1114         # set the field 'ingested' to 1 for this box
    1115         sql = "update box set ingested = 0"
    1116         try:
    1117             self.execute(sql)
    1118         except:
    1119             print "failed to clear boxes as ingested"
    1120             raise
    1121 
    1122     '''
    11231146    Inserts new box ids into the pending table
    11241147    '''
    11251148    def insertPending(self, box_id, batchType, ids):
     1149
     1150        # print "starting insert pending"
    11261151
    11271152        # first delete old pending items
    … …  
    11291154                WHERE box_id = " + str(box_id) + " \
    11301155                AND batch_type = '" + batchType + "'")
     1156
     1157        # print "deleted old items"
    11311158
    11321159        for id in ids:
    … …  
    11361163               (" + str(box_id) + ", '" + batchType + "', " + str(id) + ")"
    11371164
     1165            # print "sql: ", sql
    11381166            self.execute(sql)
    11391167
    … …  
    11681196            rs.close()
    11691197        except:
    1170             self.logger.errorPair("No free stripes", "trying to share with other client...")
     1198            self.logger.errorPair("No free stripes", "trying to steal boxes from other client...")
    11711199            sql = "SELECT ra_center, COUNT(*) AS numPending\
    11721200                   FROM box \
    … …  
    11841212                rs.close()
    11851213            except:
    1186                 self.logger.errorPair("No available stripes to share", "no stripe obtained")
     1214                self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained")
     1215
     1216        # tests against ra_center need to use a finite box
     1217        minRA = raCenter - 0.01
     1218        maxRA = raCenter + 0.01
    11871219
    11881220        self.logger.infoPair("Using RA stripe", "%f" % raCenter)
    … …  
    11901222               FROM box \
    11911223               JOIN pending ON (id = box_id) \
    1192                WHERE ra_center = " + str(raCenter) + " \
     1224               WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
    11931225               AND batch_type = '" + batchType + "' \
    11941226               AND skychunk = '" + self.skychunk.name + "' \
    … …  
    12031235
    12041236        self.registerStripe(host, pid, raCenter)
     1237
     1238        self.unlockTables()
     1239
     1240        return ids
     1241
     1242    '''
     1243    Returns ids for all boxes with pending items in a stripe of RA not already being processed
     1244    by another client. If none are available, then it uses the stripe already used by another
     1245    client that has the most pending items
     1246    '''
     1247    def getStripeBoxIdsAnyBatch(self, host, pid):
     1248
     1249        self.execute("LOCK TABLES box READ, pending READ, stripe WRITE, clients WRITE")
     1250
     1251        ids = []
     1252        raCenter = -1.0
     1253        sql = "SELECT ra_center \
     1254               FROM box \
     1255               JOIN pending ON (id = box_id) \
     1256               WHERE skychunk = '" + self.skychunk.name + "' \
     1257               AND ra_center NOT IN \
     1258               (SELECT ra_center \
     1259                FROM stripe \
     1260                WHERE skychunk = '" + self.skychunk.name + "') \
     1261               GROUP BY ra_center LIMIT 1"
     1262
     1263        try:
     1264            rs = self.executeQuery(sql)
     1265            rs.first()
     1266            raCenter = rs.getFloat(1)
     1267            rs.close()
     1268        except:
     1269            self.logger.errorPair("No free stripes", "trying to steal boxes from other client...")
     1270            sql = "SELECT ra_center, COUNT(*) AS numPending\
     1271                   FROM box \
     1272                   JOIN pending ON (id = box_id) \
     1273                   WHERE skychunk = '" + self.skychunk.name + "' \
     1274                   GROUP BY ra_center \
     1275                   ORDER BY numPending \
     1276                   DESC LIMIT 1"
     1277
     1278            try:
     1279                rs = self.executeQuery(sql)
     1280                rs.first()
     1281                raCenter = rs.getFloat(1)
     1282                rs.close()
     1283            except:
     1284                self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained")
     1285
     1286        # tests against ra_center need to use a finite box
     1287        minRA = raCenter - 0.01
     1288        maxRA = raCenter + 0.01
     1289
     1290        self.logger.infoPair("Using RA stripe", "%f" % raCenter)
     1291        sql = "SELECT DISTINCT id \
     1292               FROM box \
     1293               JOIN pending ON (id = box_id) \
     1294               WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
     1295               AND skychunk = '" + self.skychunk.name + "' \
     1296               ORDER BY dec_center DESC"
     1297
     1298        try:
     1299            rs = self.executeQuery(sql)
     1300            while (rs.next()): ids.append(rs.getInt(1))
     1301            rs.close()
     1302        except:
     1303            self.logger.errorPair("Can't get stripe boxes", sql)
    12051304
    12061305        self.unlockTables()
    … …  
    12501349        boxDim['DEC'] = rs.getFloat(2)
    12511350        boxDim['SIDE'] = rs.getFloat(3)
     1351
     1352        # SIDE is the linear full-width of the box
     1353
     1354        # XXX EAM : need to do something special at the poles...
     1355        if (boxDim['DEC'] > 85): raise
     1356
     1357        # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
     1358        dR = 0.5*boxDim['SIDE'] / math.cos(math.radians(boxDim['DEC']))
     1359        dD = 0.5*boxDim['SIDE']
     1360
     1361        boxDim['minRA']  = boxDim['RA']  - dR
     1362        boxDim['maxRA']  = boxDim['RA']  + dR
     1363        boxDim['minDEC'] = boxDim['DEC'] - dD
     1364        boxDim['maxDEC'] = boxDim['DEC'] + dD
    12521365
    12531366        return boxDim
    … …  
    13631476    Gets all items in the all_pending temporary table within the bounds of this box
    13641477    '''
    1365     def getItemsInThisThisBox(self, ra, dec):
    1366 
    1367         halfSide = self.skychunk.boxSize/2.0
    1368         minRa =  ra-halfSide
    1369         maxRa = ra+halfSide
    1370         minDec = dec-halfSide
    1371         maxDec = dec+halfSide
     1478    def getItemsInThisBox(self, minRA, maxRA, minDEC, maxDEC):
     1479
     1480        # the old linear size is ill-conceived
     1481        ## XX halfSide = self.skychunk.boxSize/2.0
     1482        ## XX minRa =  ra-halfSide
     1483        ## XX maxRa = ra+halfSide
     1484        ## XX minDec = dec-halfSide
     1485        ## XX maxDec = dec+halfSide
    13721486
    13731487        ids = []
    … …  
    13751489        sql = "SELECT DISTINCT stage_id \
    13761490               FROM all_pending \
    1377                WHERE ra_bore BETWEEN " + str(minRa) + " AND " + str(maxRa) + " \
    1378                AND dec_bore BETWEEN " + str(minDec) + " AND " + str(maxDec)
     1491               WHERE ra_bore BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
     1492               AND dec_bore BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
    13791493
    13801494        try:

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/loader.py

    r35097 r35235  
    88import sys
    99import os
     10import math
    1011import traceback
    1112import logging.config
    … …  
    4344
    4445        # connect to scratch database
    45         scratchDbs = ['1', '2', '3']
    46         for dbVersion in scratchDbs:
    47             self.scratchDb = ScratchDb(self.logger, self.config, dbVersion)
    48             if not self.config.test and self.scratchDb.anyOtherConnections():
    49                 self.logger.errorPair("This scratch Db is already in use", self.scratchDb.dbName)
    50                 self.scratchDb.disconnect()
    51                 continue
    52             break
    53 
    54         if not self.scratchDb.connected:
     46        self.scratchDb = ScratchDb(self.logger, self.config)
     47        if not self.config.test and self.scratchDb.anyOtherConnections():
     48            self.logger.errorPair("This scratch Db is already in use", self.scratchDb.dbName)
     49            self.scratchDb.disconnect()
    5550            self.exitProgram("Cannot connect to a scratch database")
    5651            raise
    … …  
    9792        except: return ret
    9893           
    99         self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb.dbName)
     94        self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
    10095
    10196        return ret
    … …  
    123118
    124119        numAttempts = 0
     120        abort = False
    125121        while True:
    126122
    127             abort = False
    128             for batchType in self.skychunk.batchTypes:
    129 
    130                 # get a stripe's worth of box IDs
    131                 boxIds = self.ippToPspsDb.getStripeBoxIds(self.HOST, self.PID, batchType)
    132 
    133                 numAttempts += 1
    134 
    135                 self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
    136                 self.logger.infoPair("numAttempts", str(numAttempts))
    137                 if len(boxIds) > 0: numAttempts = 0
    138 
    139                 for boxId in boxIds:
     123            # get a stripe's worth of box IDs
     124            boxIds = self.ippToPspsDb.getStripeBoxIdsAnyBatch(self.HOST, self.PID)
     125            self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
     126            self.logger.infoPair("numAttempts", str(numAttempts))
     127
     128            if len(boxIds) > 0: numAttempts = 0
     129            numAttempts += 1
     130
     131            for boxId in boxIds:
     132
     133                # get box info. if boxes have changed, break and start again
     134                try:
     135                    boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
     136                except:
     137                    self.logger.infoPair("Can't get details for this box", "%d" % boxId)
     138                    break
     139                self.logger.infoPair("client status", "ok")
     140
     141                for batchType in self.skychunk.batchTypes:
     142
    140143                    self.logger.infoPair("Check","client status")
    141144                    if not self.checkClientStatus():
    … …  
    143146                        break
    144147                    self.logger.infoPair("client status", "ok")
    145                     # get box info. if boxes have changed, break and start again
    146                     try:
    147                         boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
    148                     except:
    149                         self.logger.infoPair("Can't get details for this box", "%d" % boxId)
    150                         break
    151                     self.logger.infoPair("client status", "ok")
     148
    152149                    ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType)
    153                 
     150               
    154151                    # ids are the stage_ids for items to be processed (eg, stack_id for a stack CMF, cam_id for P2 smf)
    155152
    … …  
    160157                    self.logger.infoSeparator()
    161158                    self.logger.infoTitle("New box")
    162                     self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE']))
     159                    self.logger.infoPair("Box dimensions: ", "ra / dec / size : %.1f / %.1f / %.1f   --   sky bounds : (%.2f - %2.f, %2.f - %2.f)" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'], boxDim['minRA'], boxDim['maxRA'], boxDim['minDEC'], boxDim['maxDEC']))
    163160                    self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
    164                     boxSizeWithBorder = boxDim['SIDE'] + (self.skychunk.BORDER * 2)
    165                     boxSizeSansBorder = boxDim['SIDE']
    166                     self.logger.infoPair("got here", "ok")
     161
    167162                    useFullTables = 0
    168163                    if batchType != "OB":
    … …  
    171166                            # look in DVO for this box (with extra border)
    172167                            self.logger.infoPair("Querying DVO for this sky area", "")
    173                             self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
     168
     169                            # XXXX EAM : This code should not be used anymore
     170                            boxSize = 0.0
     171                            self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSize)
    174172                            #self.dvoDetections.setSkyArea()
    175173                            sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
    … …  
    191189                                useFullTables = 1
    192190                        else:
    193                             # XXX EAM : this is not currently an
    194                             # option. either remove the non-native
    195                             # ingest code or make it an option
    196                             # if we are using the native loader, always use it
    197                             # need to work out a good boundary / region strategy in coordination with
    198                             # impact of mysql insertion
    199                             if not self.ippToPspsDb.isBoxIngested(boxId):
    200                                 self.dvoDetections.nativeIngestDetections(boxId, boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)
     191                            # XXX EAM : this is not currently optional
     192                            # (it is the only path). either remove the
     193                            # non-native ingest code or make it an
     194                            # option if we are using the native
     195                            # loader, always use it need to work out a
     196                            # good boundary / region strategy in
     197                            # coordination with impact of mysql
     198                            # insertion
     199                            if not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
     200                                self.dvoDetections.nativeIngestDetections(boxId, boxDim)
    201201
    202202                            useFullTables = 1

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/metrics.py

    r33787 r35235  
    2525   
    2626        self.czarDb = CzarDb(self.logger, self.config)
    27         self.plot = Plot(self.logger, self.config, self.ippToPspsDb)
     27        self.plot = Plot(self.logger, self.skychunk, self.config, self.ippToPspsDb)
    2828   
    2929    '''
    … …  
    3737            now = datetime.datetime.now()
    3838            self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
    39             self.logger.infoPair("Config", self.config.name)
    40             self.logger.infoPair("Loading epoch", self.config.epoch)
    41             self.logger.infoPair("DVO label", self.config.dvoLabel)
    42             self.config.printBoxCoords()
     39            self.logger.infoPair("Config", self.skychunk.name)
     40            self.logger.infoPair("Loading epoch", self.skychunk.epoch)
     41            self.logger.infoPair("DVO label", self.skychunk.dvoLabel)
     42            self.skychunk.printBoxCoords()
    4343       
    4444            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    … …  
    7777       
    7878            self.checkClientStatus()
    79 
     79            self.logger.infoPair("here","yay")
    8080            # create a density plot of pending stuff for czartool
    8181            self.plot.createDensityPlot("P2", True)
    8282            self.plot.createDensityPlot("ST", True)
    8383            self.plot.createDensityPlot("OB", True)
    84 
     84            self.logger.infoPair("here","yay2")
    8585            # some clean-up before going to sleep
    8686            self.ippToPspsDb.purgeDeadClients()
    … …  
    132132           prevList = success
    133133   
    134            self.czarDb.insertStats(stage, self.config.name, batchType, numPending, numSuccess, numFail)
     134           self.czarDb.insertStats(stage, self.skychunk.name, batchType, numPending, numSuccess, numFail)
    135135   
    136136           if stage == 'processed': numPendingProcessed = numPending

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/mysql.py

    r33787 r35235  
    8484        connectionID = self.getLastConnectionID()
    8585        if connectionID == self.connectionID:
     86            print "not killing connection: ", connectionID
    8687            self.logger.debug("NOT going to kill THIS connection ID")
    8788            return
    8889
    8990        sql = "KILL %d" % connectionID
    90         self.execute(sql)
     91
     92        try:
     93            self.execute(sql)
     94        except:
     95            print "failed to kill connection: ", connectionID
    9196
    9297    '''

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/objectbatch.py

    r35097 r35235  
    5454
    5555       try:
    56            self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb.dbName)
     56           ## XX pass in the existing connection to scratchDb
     57           self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
    5758       except:
    5859           self.logger.errorPair("Unable to create instance of", "DvoObjects")
    … …  
    7778        else:
    7879            self.dvoObjects.ingestRegion(self.region)
     80
     81        cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
     82        if not self.scratchDb.tableExists(cptTableName):
     83            return False
    7984
    8085        return True
    … …  
    252257               ,dataRelease \
    253258               ,Random \
     259               ,batchID \
    254260               ) \
    255261               SELECT \
    … …  
    268274               , " + str(self.skychunk.dataRelease) + "\
    269275               , RAND() \
     276               , " + str(self.batchID) + "\
    270277               FROM " + cptTableName
    271278

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/odm.py

    r33751 r35235  
    3232        tempFile = tempfile.NamedTemporaryFile(mode='w+b')
    3333
    34         p = Popen("../perl/checkOdmStatus.pl -b " + batch + " -f " + tempFile.name,
     34        p = Popen("checkOdmStatus.pl -b " + batch + " -f " + tempFile.name,
    3535                shell=True,
    3636                stdout=PIPE,

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/plot.py

    r35097 r35235  
    1515    Constructor
    1616    '''
    17     def __init__(self, logger, skychunk, ippToPspsDb):
     17    def __init__(self, logger, skychunk, config, ippToPspsDb):
    1818
    1919        self.logger = logger
    2020        self.skychunk = skychunk
     21        self.config = config
    2122        self.ippToPspsDb = ippToPspsDb
    2223
    … …  
    2526    '''
    2627    def createDensityPlot(self, batchType, forCzartool=False):
    27 
     28       
    2829        tempFilename = "./" + self.skychunk.name + "_" + batchType + "_plotData.dat"
    2930        DATFILE = open(tempFilename,'w')
    3031        max = self.ippToPspsDb.createPendingDensityPlotData(batchType, DATFILE)
    3132        DATFILE.close()
    32 
    3333        if max == 0: max = 1
    3434
    3535        timestamp = time.strftime('%Y_%m%d_%H%M%S')
    36 
     36       
    3737        if forCzartool:
    3838            ## XXX from config??
    39             OUTPUTFILE = self.skychunk.czarPlotsPath + "/ippToPsps_density_" + self.skychunk.name + "_" + batchType + ".png"
     39            OUTPUTFILE = self.config.czarPlotsPath + "/ippToPsps_density_" + self.skychunk.name + "_" + batchType + ".png"
    4040        else:
    4141            OUTPUTFILE = self.skychunk.name + "_" + batchType + "_" + timestamp + ".png"
    42 
     42           
    4343        f = os.popen('/home/panstarrs/ipp/local/bin/gnuplot', 'w')
    44            
     44       
    4545        if 0:
    4646            TERM = "X11"
    4747        else:
    4848            TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8"
    49                      
     49        print "output to : "+ OUTPUTFILE     
    5050        print >> f, "set term " + TERM + "; \
    5151            set output \"" + OUTPUTFILE + "\"; \
    … …  
    5757            set palette rgb 22,13,10; \
    5858            set ylabel \"Dec\"; \
    59             set ylabel \"RA\"; \
     59            set xlabel \"RA\"; \
    6060            set cbrange [0:" + str(max) + "]; \
    6161            set datafile missing \"NaN\"; \

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/plotter.py

    r35097 r35235  
    1818        super(Plotter, self).__init__(argv)
    1919
    20         self.plot = Plot(self.logger, self.skychunk, self.ippToPspsDb)
     20        self.plot = Plot(self.logger, self.skychunk, self.config, self.ippToPspsDb)
    2121 
    2222        if len(argv) < 3:

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/queue.py

    r35097 r35235  
    88import sys
    99import os
     10import math
    1011import logging.config
    1112
    … …  
    1516from dvoobjects import DvoObjects
    1617from batch import Batch
     18from scratchdb import ScratchDb
    1719
    1820'''
    … …  
    2729        super(Queue, self).__init__(argv)
    2830
     31        print "inited ipptopsps"
     32
    2933        # create various objects
    3034        self.gpc1Db = Gpc1Db(self.logger, self.config)
     35        print "connect to gpc1"
     36
    3137        self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)
    3238
     39        print "made datastore"
     40
     41        # connect to scratch database
     42        self.scratchDb = ScratchDb(self.logger, self.config)
     43
     44        print "connect to scratchdb"
     45
    3346        try:
    34             self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb)
     47            self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
    3548        except:
    3649            self.exitProgram("Unable to create instance of DvoObject")
    3750            raise
     51
     52        print "got objects"
    3853
    3954        self.skychunk.printAll()
    … …  
    5671            for batchType in self.skychunk.batchTypes:
    5772
    58                 self.logger.infoTitle("Previous failed datastore loads")
    59                 self.publishAnyUnpublishedBatches(batchType)
     73                # self.logger.infoTitle("Previous failed datastore loads")
     74                # self.publishAnyUnpublishedBatches(batchType)
    6075
    6176                # get totals for whole area to check if there is anything to do
    … …  
    116131   
    117132                    # starting positions
    118                     ra = self.skychunk.minRa + self.skychunk.halfBox
    119                     dec = self.skychunk.minDec + self.skychunk.halfBox
    120    
    121                     while ra <= self.skychunk.maxRa:
    122                        while dec <= self.skychunk.maxDec:
     133
     134                    dD = 0.5*self.skychunk.boxSize
     135                    dec = self.skychunk.minDec + dD
     136                    if (dec > 85): raise
     137
     138                    # print "dec, dD: ", dec, dD
     139
     140                    # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
     141                    dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
     142                    ra = self.skychunk.minRa + dR
     143                   
     144                    # print "ra, dR: ", ra, dR
     145
     146                    # XXX not sure why this is not done with a for-loop...
     147                    while dec <= self.skychunk.maxDec:
     148                        while ra <= self.skychunk.maxRa:
    123149           
     150                           # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
     151                           minRA  = ra  - dR
     152                           maxRA  = ra  + dR
     153                           minDEC = dec - dD
     154                           maxDEC = dec + dD
     155
     156                           # this function inserts the box with given center and (linear) skychunk.boxSize
    124157                           box_id = self.ippToPspsDb.insertBox(ra, dec)
    125158           
    126                            ids = self.ippToPspsDb.getItemsInThisThisBox(ra, dec)
     159                           ids = self.ippToPspsDb.getItemsInThisBox(minRA, maxRA, minDEC, maxDEC)
    127160   
    128161                           self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |" % (
    … …  
    132165                                       len(ids)))
    133166               
     167                           # print "box_id: ", box_id
     168
    134169                           if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids)
    135 
    136                            dec = dec + self.skychunk.boxSize
    137                        dec = self.skychunk.minDec + self.skychunk.halfBox
    138                        ra = ra + self.skychunk.boxSize
    139            
     170                           # print "inserted pending"
     171                           ra = ra + 2*dR
     172                           # print "new ra: ", ra
     173
     174                        dec = dec + 2*dD
     175                        if (dec > 85): raise
     176
     177                        dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
     178                        ra = self.skychunk.minRa + dR
     179
    140180                self.logger.info("+-------------+-------------+-------------+-------------+")
    141181

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/scratchdb.py

    r35097 r35235  
    1919    Constructor
    2020    '''
    21     def __init__(self, logger, config, dbVersion, dbName=None):
     21    def __init__(self, logger, config):
    2222
    2323        # create gpc1 database objects
     24        # XXX the test option is probably not needed since
     25        # the scratchDb config infor is set up specially
    2426        if (config.test):
    2527            dbType = "localdatabase_test"
    … …  
    2729            dbType = "localdatabase"
    2830
    29         if not dbName:
    30             dbName = config.getDbName(dbType)
    31             dbName += dbVersion
     31        print "trying to connect: ", dbType
     32
     33        dbName = config.getDbName(dbType)
    3234
    3335        super(ScratchDb, self).__init__(logger, config, dbType, dbName)
    … …  
    432434    def purgeTheseDvoRegions(self, regions, fileTypes):
    433435
    434         detectionCount = self.getRowCount(self.dvoDetectionTable)
     436        # XXX this must have been a remnant from something else: we are deleting a region, not entries from the dvoDetectionTable
     437        # detectionCount = self.getRowCount(self.dvoDetectionTable)
    435438        for region in regions:
    436439         
    … …  
    448451                    pass
    449452
    450         deletedDetectionCount = detectionCount - self.getRowCount(self.dvoDetectionTable)
     453        # deletedDetectionCount = detectionCount - self.getRowCount(self.dvoDetectionTable)
    451454
    452455        # after a big detection delete, we should OPTIMIZE the table
    453         if deletedDetectionCount > 0:
    454             self.logger.infoPair("Running OPTIMIZE on ", self.dvoDetectionTable)
    455             self.optimizeTable(self.dvoDetectionTable)
    456 
    457         return deletedDetectionCount
     456        # if deletedDetectionCount > 0:
     457        #     self.logger.infoPair("Running OPTIMIZE on ", self.dvoDetectionTable)
     458        #     self.optimizeTable(self.dvoDetectionTable)
     459
     460        return 1
    458461           
    459462    '''

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/setupScratchDb.py

    r35097 r35235  
    5757        self.config = Config()
    5858
    59         if len(argv) < 2:
     59        if len(argv) != 1:
    6060            self.printUsage()
    6161            self.exitProgram("incorrect args")
    … …  
    6565        self.PID = os.getpid()
    6666
     67        self.config.setScratchDbLeaf(self.HOST)
     68
    6769        # XXX Shouldn't config set up logger?
    6870        # (not yet, since we set HOST & PID with program...)
    … …  
    7072
    7173        try:
    72             self.scratchDb = ScratchDb(self.logger, self.config, argv[1])
     74            self.scratchDb = ScratchDb(self.logger, self.config)
    7375        except:
    7476            self.exitProgram("Could not connect to a scratch Db")
    … …  
    100102    def printUsage(self):
    101103        # write message to log or stdout if no logger set up
    102         msg = os.path.basename(sys.argv[0]) + " <scratchdb_version> "
     104        msg = os.path.basename(sys.argv[0]) + " [-test]"
    103105        try:
    104106            self.logger.errorPair("Usage:", msg)

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/skychunk.py

    r35097 r35235  
    3838
    3939        # XXX this is a poor place to put this information -- it is completely gpc-specific. 
    40         # probably should go elsewhere
     40        # probably should go elsewhere.  This number is scaled in loader.py by cos(dec)
    4141        self.BORDER = 1.65
    4242        self.isLoaded = False

  • branches/eam_branches/ipp-20130207/ippToPsps/jython/stackbatch.py

    r35097 r35235  
    420420        )"
    421421        self.scratchDb.execute(sql)
    422 
     422        self.scratchDb.updateAllRows("StackMeta", "batchID", str(self.batchID))
    423423        self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID))
    424424        self.scratchDb.updateFilterID("StackMeta", self.filter)
    … …  
    846846        #self.scratchDb.makeColumnUnique("StackDetection", "objID")
    847847        self.scratchDb.createIndex("StackDetection", "ippDetectID")
     848        self.scratchDb.createIndex("StackDetection", "stackDetectID")
    848849        self.scratchDb.createIndex("StackApFlx", "ippDetectID")
    849850        self.scratchDb.createIndex("StackModelFit", "ippDetectID")

  • branches/eam_branches/ipp-20130207/ippToPsps/perl

    • Property svn:ignore set to
      Makefile
      Makefile.in

  • branches/eam_branches/ipp-20130207/ippToPsps/test/fulltest.sh

    r35097 r35235  
    2525# mkgpc1.sh user localhost eam eam
    2626
    27 set mkgpc1       = 0
     27set mkgpc1       = 1
    2828set initdb       = 1
    29 set mkdummy      = 1
    3029set initscratch  = 1
    3130set camqueue     = 1
    … …  
    3433set stackqueue   = 1
    3534set stackbatch   = 1
    36 set objectqueue  = 0
    37 set objectbatch  = 0
     35set objectqueue  = 1
     36set objectbatch  = 1
    3837
    3938set OUTDIR = "testdata"
    … …  
    5453endif
    5554
    56 ## XXX no longer needed : setupScratchDb does not need a fake 'config' (or 'chunk')
    57 ## # create a dummy config so setupScratchDb.py does not fail
    58 ## if ($mkdummy) then
    59 ##   ippjython queue.py -test edit <<EOF
    60 ## dummy
    61 ## none
    62 ## IPP_PSPS
    63 ## 0
    64 ## none
    65 ## none
    66 ## 0
    67 ## 0
    68 ## 0
    69 ## 0
    70 ## 1
    71 ## none
    72 ## 0
    73 ## 0
    74 ## 1
    75 ## 1
    76 ## 2001-01-01 00:00:00.0
    77 ## NONE
    78 ## NONE
    79 ## 1
    80 ## 0
    81 ## 0
    82 ## 1
    83 ## 0
    84 ## EOF
    85 ## endif
    86 
    8755# re-create the ipptopsps database
    8856if ($initscratch) then
    8957
    9058  # create the database (ignore failure if it already exists)
    91   mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_test_scratch1"
    92   # mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_scratch2"
    93   # mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_scratch3"
    94 
    95   mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test_scratch1"
    96   # mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_scratch2"
    97   # mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_scratch3"
    98 
    99   # setupScratchDb.py now takes the scratch name version, which is appended to the
    100   # name in the settings.xml table
    101   ippjython setupScratchDb.py -test 1
    102   # ippjython setupScratchDb.py -test 2
    103   # ippjython setupScratchDb.py -test 3
     59  mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_test_scratch"
     60  mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test_scratch"
     61  ippjython setupScratchDb.py -test
    10462endif
    10563

  • branches/eam_branches/ipp-20130207/ippToPsps/test/mkgpc1.sh

    r35097 r35235  
    6161  CREATE TABLE camProcessedExp (
    6262    path_base CHAR(255),
    63     cam_id INT
     63    cam_id INT,
     64    fault INT
    6465  );
    6566  CREATE TABLE fakeRun (
    … …  
    9899  );
    99100  CREATE TABLE addRun (
     101    add_id INT,
    100102    stage CHAR(64),
    101103    state CHAR(64),
    102104    stage_id INT,
    103105    minidvodb_name CHAR(64)
     106  );
     107  CREATE TABLE addProcessedExp (
     108    add_id INT,
     109    fault INT
    104110  );
    105111  CREATE TABLE minidvodbRun (
    … …  
    148154  set ntable = `echo "USE $dbname; show tables;" | mysql -h $dbhost -u $dbuser $dbpass | wc -l`
    149155  echo ntable: $ntable
    150   if ($ntable != 20) then
    151     echo "expected 20 lines (19 tables), but got $ntable, not deleting"
     156  if ($ntable != 21) then
     157    echo "expected 21 lines (19 tables), but got $ntable, not deleting"
    152158    exit 2
    153159  endif

  • branches/eam_branches/ipp-20130207/ippToPsps/test/mkgpc1data.dvo

    r35123 r35235  
    291291  dbinsert chipRun (exp_id, chip_id) values ($myID, $myID)
    292292  dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($myID, $myID, "testgroup", "testsoft")
    293   dbinsert camProcessedExp (cam_id, path_base) values ($myID, "$filename")
     293  dbinsert camProcessedExp (cam_id, path_base, fault) values ($myID, "$filename", 0)
    294294  dbinsert fakeRun (fake_id, cam_id) values ($myID, $myID)
    295295  dbinsert warpRun (warp_id, fake_id) values ($myID, $myID)
    296296  dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($myID, "skycell.0000.000", "XY00")
    297297
    298   dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("cam", "full", $myID, "catdir.cam")
     298  dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($myID, "cam", "full", $myID, "catdir.cam")
     299  dbinsert addProcessedExp (add_id, fault) values ($myID, 0)
    299300end
    300301

  • branches/eam_branches/ipp-20130207/ippToPsps/test/sastest.sh

    r35097 r35235  
    1919# mkgpc1.sh user localhost eam eam
    2020
    21 set mkgpc1      = 0
    22 set initdb      = 0
    23 set mkdummy     = 0
    24 set initscratch = 0
     21set mkgpc1      = 1
     22set initdb      = 1
     23set mkdummy     = 1
     24set initscratch = 1
    2525set initbatch   = 1
    2626set camqueue    = 1
    2727set cambatch    = 1
    28 set stackqueue  = 1
    29 set stackbatch  = 1
     28set stackqueue  = 0
     29set stackbatch  = 0
    3030
    3131set OUTDIR = "sas12"
    … …  
    4646endif
    4747
    48 # create a dummy config so setupScratchDb.py does not fail
    49 if ($mkdummy) then
    50   ippjython queue.py -test edit <<EOF
    51 dummy
    52 none
    53 IPP_PSPS
    54 0
    55 none
    56 none
    57 0
    58 0
    59 0
    60 0
    61 1
    62 none
    63 0
    64 0
    65 1
    66 1
    67 2001-01-01 00:00:00.0
    68 NONE
    69 NONE
    70 1
    71 0
    72 0
    73 1
    74 0
    75 1
    76 EOF
    77 endif
    78 
    7948# re-create the ipptopsps database
    8049if ($initscratch) then
    8150
    8251  # create the database (ignore failure if it already exists)
    83   mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_scratch1"
    84   # mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_scratch2"
    85   # mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_scratch3"
    86 
    87   mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_scratch1"
    88   # mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_scratch2"
    89   # mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_scratch3"
    90 
    91   ippjython setupScratchDb.py -test dummy ipptopsps_scratch1
    92   # ippjython setupScratchDb.py -test dummy ipptopsps_scratch2
    93   # ippjython setupScratchDb.py -test dummy ipptopsps_scratch3
     52  mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_scratch"
     53  mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_scratch"
     54  ippjython setupScratchDb.py -test
    9455endif
    9556
    … …  
    117783PI
    118791
    119 0
    120 0
     801
     811
    121821
    122830
Note: See TracChangeset for help on using the changeset viewer.