IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Apr 3, 2013, 12:52:33 PM (13 years ago)
Author:
eugene
Message:

changed dvoDetectionFull mysql engine to 'MEMORY' to improve ingest speed

handle errrors in calls to dvopsps (only in detectionbatch.py)

removed a lot of excess verbosity used for debugging / testing

determine the mysql major version number via mysql query

added test option -realgpc to use test ipptopsps databases with a real gpc1 dataset

add version-dependent call to 'set session old_alter_table' to allow
the ALTER IGNORE statements to work on mysql v5.5 machines (like our
desktops).

raise an exception in case the load infile operations fail

remove unneeded 'killLastConnectionID'

extend logging time to milliseconds

allow an exposure to be skipped without aborting loader

check that a P2 image exists in the dvoImage table (otherwise skip the chip)

change the mysql floating point types for the dvoDetection table to use FLOAT/DOUBLE instead of REAL/FLOAT

remove the old test for valid astrometry of a chip (NASTRO >= 50)

this test was crude and not really the right test. it was also used
to verify that an image should / should not be included. now we check
against dvoImages to see if the chip should be loaded or not (DVO
rejects input images for bad astrometry)

skip an exposure if too many detections have been rejected. this is a sign of a problem and should require more attention

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20130307/ippToPsps/test/fulltest.sh

    r35225 r35352  
    11#!/bin/csh -f
    2 
    3 set stackqueuename = 0
    4 if ($#argv > 0) then
    5   set stackqueuename = $1
    6   echo "stackqueue: $stackqueuename"
    7 endif
    8 
    9 set objectqueuename = 0
    10 if ($#argv > 0) then
    11   set stackqueuename = $1
    12   echo "stackqueue: $stackqueuename"
    13 endif
    142
    153# a bit crude : get the datadir path by interpollation of the PATH element for psconfig
    … …  
    2412# before running this test, you must have a mysql user:
    2513# mkgpc1.sh user localhost eam eam
    26 
    2714set mkgpc1       = 1
    2815set initdb       = 1
    2916set initscratch  = 1
     17set initbatch    = 1
    3018set camqueue     = 1
    31 set initbatch    = 1
    3219set cambatch     = 1
    3320set stackqueue   = 1
    … …  
    3623set objectbatch  = 1
    3724
     25set args    = ""
     26
     27while ("$1" != "")
     28 switch ("$1")
     29  case -mkgpc1
     30    set mkgpc1 = 0
     31    breaksw;
     32  case -initdb
     33    set initdb = 0
     34    breaksw;
     35  case -initscratch
     36    set initscratch = 0
     37    breaksw;
     38  case -initbatch
     39    set initbatch = 0
     40    breaksw;
     41  case -camera
     42    set camqueue = 0
     43    set cambatch = 0
     44    breaksw;
     45  case -stack
     46    set stackqueue = 0
     47    set stackbatch = 0
     48    breaksw;
     49  case -object
     50    set objectqueue = 0
     51    set objectbatch = 0
     52    breaksw;
     53  default:
     54    set args=($args $1);
     55    breaksw;
     56  endsw
     57  shift
     58end
     59
     60set stackqueuename = 0
     61set objectqueuename = 0
     62
    3863set OUTDIR = "testdata"
    3964set queuename = "test"
    4065
    4166if ($mkgpc1) then
     67  echo ""
     68  echo " ---- mkgpc1data.dvo : creating the dvo & gpc1 database entries ----"
    4269  mkgpc1data.dvo
    4370endif
    … …  
    4572# create the ipptopsps database
    4673if ($initdb) then
     74  echo ""
     75  echo " ---- init ipptopsps_test ----"
    4776
    4877  # does the database already exist?
    … …  
    5584# re-create the ipptopsps database
    5685if ($initscratch) then
     86  echo ""
     87  echo " ---- init ipptopsps_test_scratch ----"
    5788
    5889  # create the database (ignore failure if it already exists)
    … …  
    6293endif
    6394
     95# this is poor : we need to queue something in order to generate an init batch
     96# this is because a skychunk carries the info about the datastore used to publish
     97# the batch
     98if ($initbatch) then
     99  echo ""
     100  echo " ---- make skychunk for IN batches ----"
     101
     102  ippjython queue.py -test edit <<EOF
     103initbatch
     104test_ds
     105IPP_PSPS
     1060
     107none
     108none
     1090
     1100
     1110
     1120
     1132
     114none
     1150
     1160
     1171
     1181
     1192012-12-10 00:00:00.0
     1203PI
     1213PI
     1220
     1230
     1240
     1251
     1260
     127y
     128y
     129EOF
     130endif
     131
     132  echo ""
     133  echo " ---- create init batch ----"
     134
     135  rm -rf $OUTDIR/IN
     136  ippjython loader.py -test initbatch init
     137endif
     138
    64139# this code is sensitive to a previously existing database?
    65140if ($camqueue) then
     141  echo ""
     142  echo " ---- queue P2 batches ----"
     143
    66144  ippjython queue.py -test edit <<EOF
    67145$queuename
    … …  
    94172endif
    95173
    96 if ($initbatch) then
    97   rm -rf $OUTDIR/IN
    98   ippjython loader.py -test $queuename init
    99 endif
    100 
    101174if ($cambatch) then
     175  echo ""
     176  echo " ---- load P2 batches ----"
     177
    102178  rm -rf $OUTDIR/P2
    103179  ippjython loader.py -test $queuename once
    … …  
    106182# this code is sensitive to a previously existing database?
    107183if ($stackqueue) then
     184  echo ""
     185  echo " ---- queue ST batches ----"
     186
    108187  if ($stackqueuename == 0) then
    109188    set stackqueuename = $queuename\_stk
    … …  
    140219
    141220if ($stackbatch) then
     221  echo ""
     222  echo " ---- load ST batches ----"
     223
    142224  if ($stackqueuename == 0) then
    143225    set stackqueuename = $queuename\_stk
    … …  
    149231# this code is sensitive to a previously existing database?
    150232if ($objectqueue) then
     233  echo ""
     234  echo " ---- queue OB batches ----"
     235
    151236  if ($objectqueuename == 0) then
    152237    set objectqueuename = $queuename\_obj
    … …  
    183268
    184269if ($objectbatch) then
     270  echo ""
     271  echo " ---- load OB batches ----"
     272
    185273  if ($objectqueuename == 0) then
    186274    set objectqueuename = $queuename\_obj
Note: See TracChangeset for help on using the changeset viewer.