Index: /trunk/ippToPsps/jython/diffbatch.py =================================================================== --- /trunk/ippToPsps/jython/diffbatch.py (revision 39014) +++ /trunk/ippToPsps/jython/diffbatch.py (revision 39015) @@ -114,5 +114,5 @@ self.difftypeID[num] = meta[1]; - self.ippDiffID[num] = stage_id + self.ippDiffID[num] = stageid self.posImageID[num] = meta[2]; self.negImageID[num] = meta[3]; @@ -122,5 +122,6 @@ ## MEH -- should be able to use diff header now for mjd (see below) #self.obsTime[num] = meta[7]; - self.expID[num] = meta[8]; + # tess_id is handled below, that's #8. + self.expID[num] = meta[9]; self.logger.infoPair("loading diff stage cmf", diffSkyFileID) self.fits[num] = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, int(diffSkyFileID)) @@ -160,4 +161,5 @@ self.dropTableVerbose("DiffToImage_"+str(num)) self.dropTableVerbose("DiffDetection_"+str(num)) + self.dropTableVerbose("DiffDetEffMeta_"+str(num)) if not self.analysisVer[num]: self.analysisVer[num] = -999 @@ -440,4 +442,83 @@ + ''' + Populate DiffDetEffMeta + ''' + def populateDiffDetEffMeta(self,num): + diffSkyFileID = self.number[num] + self.logger.infoPair("Processing table", "DiffDetEffMeta") + + if diffSkyFileID <= 0: + self.logger.infoPair("no diff data for number", num) + return False + + cmfTable = "SkyChip_deteff_" + str(num) + if not cmfTable in self.tablesLoaded: + self.logger.infoPair("no deteff data for diff", diffSkyFileID) + self.logger.infoPair("(did not find table in tablesLoaded)", cmfTable) + for name in self.tablesLoaded: + print "-- table: " + name + os._exit(3) + return False + + self.logger.infoPair("Found cmfTable:", cmfTable) + + ippTableName = "DiffDetEffMeta_" + str(num) + sql = "CREATE TABLE " + ippTableName + " LIKE DiffDetEffMeta" + try: self.scratchDb.execute(sql) + except: pass + + self.logger.infoPair("Created tmp table:", ippTableName) + + header = self.header[num] + # CZW: this is broken, and can't get the zeropoint + # zpImage = self.scratchDb.getImageZeroPoint(diffSkyFileID) + zpImage = 0.0 + + if zpImage < 0.0: + zpImage = 0.0 + if zpImage == 0.0: + self.logger.infoPair("CZW If this value for the zeropoint is zero, the getImageZeroPoint call is likely still broken", str(zpImage)) + + + magref = self.getKeyFloat(header,"%.8f","DETEFF.MAGREF") + zpImage + self.logger.infoPair("Found zeropoint and set magref", str(magref)) + self.logger.infoPair("CZW If that magref is -5.804804, then it's reading from the PHU, not the Deteff extension.","In a test, I mean.") + sqlLine = sqlUtility("INSERT INTO " + ippTableName + "(") + sqlLine.group("diffImageID", str(diffSkyFileID)) + sqlLine.group("magref", magref) + sql = sqlLine.make(") VALUES ( ", ")") + + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql: ', sql) + raise + + self.logger.infoPair("CZW:", "About to loop over deteff bins.") + + for i in xrange(1, 14): + stringint = str(i) + if i < 10: + stringint = '0' + str(i) + + sql = "UPDATE " + ippTableName + " , " + cmfTable + \ + " SET offset"+stringint + " = OFFSET, counts" \ + +stringint + " = COUNTS, diffMean" \ + +stringint + " = DIFF_MEAN, diffStdev" \ + +stringint + " = DIFF_STDEV, errMean" \ + +stringint + " = ERR_MEAN WHERE table_index = " + str(i) + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('filed sql: ', sql) + raise + # end for + # Append this like this? + self.tablesToExport.append(ippTableName) + # end filters + + + return True @@ -452,5 +533,5 @@ self.logger.debug("Altering PSPS tables") self.logger.debug("Creating indexes on PSPS tables") - self.scratchDb.makeColumnUnique("DiffDetection", "objID") + self.scratchDb.makeColumnUnique("DiffDetection", "diffObjID") self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID") @@ -555,5 +636,7 @@ self.setMinMaxDiffObjID(["DiffDetection_"+str(num)]) - + self.populateDiffDetEffMeta(num) + self.logger.infoPair("populated","DiffDetEffMeta_"+str(num)) + # CZW: I appended to tablesToExport inside populateDiffDetEffMeta to be like stack. #test = self.scratchDb.getRowCount("Object") #return False @@ -623,5 +706,6 @@ self.haveApNpix[num] = 1 - myTable = table.name + "_" + str(num) + + myTable = table.name.replace(".","_") + "_" + str(num) count = count + 1 Index: /trunk/ippToPsps/jython/gpc1db.py =================================================================== --- /trunk/ippToPsps/jython/gpc1db.py (revision 39014) +++ /trunk/ippToPsps/jython/gpc1db.py (revision 39015) @@ -433,5 +433,5 @@ def getDiffStageMeta(self, diffSkyfileID): - self.logger.debug("Querying GPC1 for stack meta data") + self.logger.debug("Querying GPC1 for diff meta data") meta = [] @@ -528,5 +528,5 @@ JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \ AND diff_id = stage_id AND stage = 'diff') \ - JOIN diffSkyfile using (diff_id, skycell_id) \ + JOIN diffSkyfile using (diff_id, skycell_id, diff_skyfile_id) \ WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \ AND minidvodbRun.state = 'merged' \ @@ -557,5 +557,9 @@ files.append(i.rstrip()) print "append" - print i +# No clue what this is here for, but it breaks things. +# print i + else: + files = glob.glob(pathBase + ".cmf") + if len(files) < 1: return None fits = Fits(self.logger, self.config, files[0]) Index: /trunk/ippToPsps/test/fulltest.sh =================================================================== --- /trunk/ippToPsps/test/fulltest.sh (revision 39014) +++ /trunk/ippToPsps/test/fulltest.sh (revision 39015) @@ -1,3 +1,8 @@ #!/bin/csh -f + +set dbuser = 'dvo' +set dbpass = '-pdvo' +# set dbuser = 'czw' +# set dbpass = '-pczw' chmod +x mkgpc1data.dvo mkgpc1.sh @@ -29,4 +34,9 @@ set forcedobjqueue = 1 set forcedobjbatch = 1 +set diffqueue = 1 +set diffbatch = 1 +set diffobjqueue = 1 +set diffobjbatch = 1 + set options = "-test" @@ -42,5 +52,5 @@ echo " -XX : turn off features" echo " +XX : turn on features" - echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, forcedwarp, object, forcedobject" + echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, forcedwarp, object, forcedobject, diff, diffobj" exit 2 case -all @@ -59,4 +69,8 @@ set forcedobjqueue = 0 set forcedobjbatch = 0 + set diffqueue = 0 + set diffbatch = 0 + set diffobjqueue = 0 + set diffobjbatch = 0 breaksw; case -mkgpc1 @@ -91,4 +105,12 @@ set forcedobjqueue = 0 set forcedobjbatch = 0 + breaksw; + case -diff + set diffqueue = 0 + set diffbatch = 0 + breaksw; + case -diffobj + set diffobjqueue = 0 + set diffobjbatch = 0 breaksw; case -debug @@ -125,4 +147,8 @@ set forcedobjqueue = 1 set forcedobjbatch = 1 + set diffqueue = 1 + set diffbatch = 1 + set diffobjqueue = 1 + set diffobjbatch = 1 breaksw; case +mkgpc1 @@ -157,4 +183,12 @@ set forcedobjqueue = 1 set forcedobjbatch = 1 + breaksw; + case +diff + set diffqueue = 1 + set diffbatch = 1 + breaksw; + case +diffobj + set diffobjqueue = 1 + set diffobjbatch = 1 breaksw; default: @@ -169,5 +203,5 @@ echo " -XX : turn off features" echo " +XX : turn on features" - echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, object, stack, forcedwarp, forcedobject" + echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, object, stack, forcedwarp, forcedobject, diff, diffobj" exit 2 endif @@ -179,4 +213,7 @@ set forcedwarpqueuename = $queuename\_wrp set forcedobjqueuename = $queuename\_fwo +set diffqueuename = $queuename\_dif +set diffobjqueuename = $queuename\_dio + if ($mkgpc1) then @@ -197,6 +234,6 @@ # does the database already exist? # XXX this command will blow away an existing database!! - mysql -h localhost -u dvo -pdvo -e "create database if not exists ipptopsps_test" || exit 3 - mysql -h localhost -u dvo -pdvo ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql || exit 3 + mysql -h localhost -u $dbuser $dbpass -e "create database if not exists ipptopsps_test" || exit 3 + mysql -h localhost -u $dbuser $dbpass ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql || exit 3 echo " ---- init ipptopsps_test: done ----" endif @@ -208,6 +245,6 @@ # create the database (ignore failure if it already exists) - mysql -h localhost -u dvo -pdvo -e "drop database if exists ipptopsps_test_scratch" - mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test_scratch" + mysql -h localhost -u $dbuser $dbpass -e "drop database if exists ipptopsps_test_scratch" + mysql -h localhost -u $dbuser $dbpass -e "create database ipptopsps_test_scratch" echo ippjython setupScratchDb.py $options ippjython setupScratchDb.py $options < /dev/null @@ -527,2 +564,108 @@ endif + +############################ DF batches ################################ + +# this code is sensitive to a previously existing database? +if ($diffqueue) then + echo "" + echo " ---- queue DF batches ----" + + echo "$diffqueuename " > diffbatch.dat # skychunk name + echo "test_ds " >> diffbatch.dat # datastore_product + echo "IPP_PSPS " >> diffbatch.dat # datastore_type + echo "0 " >> diffbatch.dat # datastore_publish + echo "catdir.dif " >> diffbatch.dat # dvo_label + echo "$OUTDIR/catdir.dif " >> diffbatch.dat # dvo_location + echo "9 " >> diffbatch.dat # min_ra + echo "11 " >> diffbatch.dat # max_ra + echo "19 " >> diffbatch.dat # min_dec + echo "21 " >> diffbatch.dat # max_dec + echo "2 " >> diffbatch.dat # box_size + echo "$OUTDIR " >> diffbatch.dat # base_path + echo "0 " >> diffbatch.dat # data_release + echo "0 " >> diffbatch.dat # delete_local + echo "1 " >> diffbatch.dat # delete_datastore + echo "1 " >> diffbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> diffbatch.dat # epoch + echo "3PI " >> diffbatch.dat # survey + echo "3PI " >> diffbatch.dat # psps_survey + echo "0 " >> diffbatch.dat # queue_P2 + echo "0 " >> diffbatch.dat # queue_ST + echo "0 " >> diffbatch.dat # queue_OB + echo "1 " >> diffbatch.dat # queue_DF + echo "0 " >> diffbatch.dat # queue_DO + echo "0 " >> diffbatch.dat # queue_FW + echo "0 " >> diffbatch.dat # queue_FO + echo "1 " >> diffbatch.dat # active + echo "0 " >> diffbatch.dat # parallel + echo "use_new " >> diffbatch.dat # P2_smf_version + echo "any " >> diffbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> diffbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> diffbatch.dat # trange_end + ippjython queue.py $options edit -simtest < diffbatch.dat + if ($status) exit 1 +endif + +if ($diffbatch) then + echo "" + echo " ---- load DF batches ----" + + rm -rf $OUTDIR/FW + ippjython loader.py $options $diffqueuename once -simtest < /dev/null + if ($status) exit +endif + +############################ DO batches ################################ + +# this code is sensitive to a previously existing database? +if ($diffobjqueue) then + echo "" + echo " ---- queue do batches ----" + + echo "$diffobjqueuename " > diffobjbatch.dat # skychunk name + echo "test_ds " >> diffobjbatch.dat # datastore_product + echo "IPP_PSPS " >> diffobjbatch.dat # datastore_type + echo "0 " >> diffobjbatch.dat # datastore_publish + echo "catdir.dif " >> diffobjbatch.dat # dvo_label + echo "$OUTDIR/catdir.dif " >> diffobjbatch.dat # dvo_location + echo "9 " >> diffobjbatch.dat # min_ra + echo "11 " >> diffobjbatch.dat # max_ra + echo "19 " >> diffobjbatch.dat # min_dec + echo "21 " >> diffobjbatch.dat # max_dec + echo "2 " >> diffobjbatch.dat # box_size + echo "$OUTDIR " >> diffobjbatch.dat # base_path + echo "0 " >> diffobjbatch.dat # data_release + echo "0 " >> diffobjbatch.dat # delete_local + echo "1 " >> diffobjbatch.dat # delete_datastore + echo "1 " >> diffobjbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> diffobjbatch.dat # epoch + echo "3PI " >> diffobjbatch.dat # survey + echo "3PI " >> diffobjbatch.dat # psps_survey + echo "0 " >> diffobjbatch.dat # queue_P2 + echo "0 " >> diffobjbatch.dat # queue_ST + echo "0 " >> diffobjbatch.dat # queue_OB + echo "0 " >> diffobjbatch.dat # queue_DF + echo "1 " >> diffobjbatch.dat # queue_DO + echo "0 " >> diffobjbatch.dat # queue_FW + echo "0 " >> diffobjbatch.dat # queue_FO + echo "1 " >> diffobjbatch.dat # active + echo "0 " >> diffobjbatch.dat # parallel + echo "use_new " >> diffobjbatch.dat # P2_smf_version + echo "any " >> diffobjbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> diffobjbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> diffobjbatch.dat # trange_end + ippjython queue.py $options edit once -simtest < diffobjbatch.dat + if ($status) exit +endif + +if ($diffobjbatch) then + echo "" + echo " ---- load LO batches ----" + + rm -rf $OUTDIR/LO + + ippjython loader.py $options $diffobjqueuename once -simtest < /dev/null + if ($status) exit +endif + Index: /trunk/ippToPsps/test/gpc1schema.sql =================================================================== --- /trunk/ippToPsps/test/gpc1schema.sql (revision 39014) +++ /trunk/ippToPsps/test/gpc1schema.sql (revision 39015) @@ -1,5 +1,5 @@ CREATE TABLE ippToPspsFake ( my_id INT ); - CREATE TABLE skycell ( skycell_id CHAR(64), radeg REAL, decdeg REAL ); + CREATE TABLE skycell ( tess_id CHAR(64), skycell_id CHAR(64), radeg REAL, decdeg REAL ); CREATE TABLE rawExp ( exp_id INT, exp_name CHAR(64), exp_time REAL, filter CHAR(64), bg REAL, bg_stdev REAL, ra FLOAT, decl FLOAT, dateobs DATETIME ); CREATE TABLE chipRun ( exp_id INT, chip_id INT ); @@ -19,4 +19,8 @@ CREATE TABLE skycalResult ( skycal_id INT, path_base CHAR(255)); + CREATE TABLE diffRun ( diff_id INT, tess_id CHAR(64), state CHAR(16), diff_mode INT, software_ver CHAR(16) ); + CREATE TABLE diffInputSkyfile ( diff_id INT, warp1 INT, stack2 INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64) ); + CREATE TABLE diffSkyfile ( diff_id INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64), path_base CHAR(255) ); + CREATE TABLE addRun ( add_id INT, stage CHAR(64), stage_extra1 INT, state CHAR(64), stage_id INT, minidvodb_name CHAR(64) ); CREATE TABLE addProcessedExp ( add_id INT, fault INT ); Index: /trunk/ippToPsps/test/mkgpc1.sh =================================================================== --- /trunk/ippToPsps/test/mkgpc1.sh (revision 39014) +++ /trunk/ippToPsps/test/mkgpc1.sh (revision 39015) @@ -58,6 +58,6 @@ set ntable = `mysql -B -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" | wc -l` echo ntable: $ntable - if ($ntable != 25) then - echo "expected 25 lines (24 tables), but got $ntable, not deleting" + if ($ntable != 28) then + echo "expected 28 lines (27 tables), but got $ntable, not deleting" exit 2 endif Index: /trunk/ippToPsps/test/mkgpc1data.dvo =================================================================== --- /trunk/ippToPsps/test/mkgpc1data.dvo (revision 39014) +++ /trunk/ippToPsps/test/mkgpc1data.dvo (revision 39015) @@ -23,29 +23,29 @@ # imagedata describes the full fake exposure set. the fields are list imagedata -# exp_id stk_id filebase M_off date time warpfile filter - 1 1 $OUTDIR/test.01 0.000 2010/01/01 01:00:00 $OUTDIR/warp.01 g - 2 1 $OUTDIR/test.02 -0.025 2010/01/01 02:00:00 $OUTDIR/warp.02 g - 3 1 $OUTDIR/test.03 0.025 2010/01/01 03:00:00 $OUTDIR/warp.03 g - 4 1 $OUTDIR/test.04 0.010 2010/01/01 04:00:00 $OUTDIR/warp.04 g - 5 2 $OUTDIR/test.05 0.000 2010/01/01 05:00:00 $OUTDIR/warp.05 g - 6 2 $OUTDIR/test.06 -0.025 2010/01/01 06:00:00 $OUTDIR/warp.06 g - 7 2 $OUTDIR/test.07 0.025 2010/01/01 07:00:00 $OUTDIR/warp.07 g - 8 2 $OUTDIR/test.08 0.010 2010/01/01 08:00:00 $OUTDIR/warp.08 g - 9 3 $OUTDIR/test.09 0.000 2010/01/01 09:00:00 $OUTDIR/warp.09 g - 10 3 $OUTDIR/test.10 -0.025 2010/01/01 10:00:00 $OUTDIR/warp.10 g - 11 3 $OUTDIR/test.11 0.025 2010/01/01 11:00:00 $OUTDIR/warp.11 g - 12 3 $OUTDIR/test.12 0.010 2010/01/01 12:00:00 $OUTDIR/warp.12 g - 13 4 $OUTDIR/test.13 -0.025 2010/01/02 01:00:00 $OUTDIR/warp.13 r - 14 4 $OUTDIR/test.14 0.025 2010/01/02 02:00:00 $OUTDIR/warp.14 r - 15 4 $OUTDIR/test.15 0.010 2010/01/02 03:00:00 $OUTDIR/warp.15 r - 16 4 $OUTDIR/test.16 0.000 2010/01/02 04:00:00 $OUTDIR/warp.16 r - 17 5 $OUTDIR/test.17 -0.025 2010/01/02 05:00:00 $OUTDIR/warp.17 r - 18 5 $OUTDIR/test.18 0.025 2010/01/02 06:00:00 $OUTDIR/warp.18 r - 19 5 $OUTDIR/test.19 0.010 2010/01/02 07:00:00 $OUTDIR/warp.19 r - 20 5 $OUTDIR/test.20 0.000 2010/01/02 08:00:00 $OUTDIR/warp.20 r - 21 6 $OUTDIR/test.21 -0.025 2010/01/02 09:00:00 $OUTDIR/warp.21 r - 22 6 $OUTDIR/test.22 0.025 2010/01/02 10:00:00 $OUTDIR/warp.22 r - 23 6 $OUTDIR/test.23 0.010 2010/01/02 11:00:00 $OUTDIR/warp.23 r - 24 6 $OUTDIR/test.24 -0.020 2010/01/02 12:00:00 $OUTDIR/warp.24 r +# exp_id stk_id filebase M_off date time warpfile filter difffile + 1 1 $OUTDIR/test.01 0.000 2010/01/01 01:00:00 $OUTDIR/warp.01 g $OUTDIR/diff.01 + 2 1 $OUTDIR/test.02 -0.025 2010/01/01 02:00:00 $OUTDIR/warp.02 g $OUTDIR/diff.02 + 3 1 $OUTDIR/test.03 0.025 2010/01/01 03:00:00 $OUTDIR/warp.03 g $OUTDIR/diff.03 + 4 1 $OUTDIR/test.04 0.010 2010/01/01 04:00:00 $OUTDIR/warp.04 g $OUTDIR/diff.04 + 5 2 $OUTDIR/test.05 0.000 2010/01/01 05:00:00 $OUTDIR/warp.05 g $OUTDIR/diff.05 + 6 2 $OUTDIR/test.06 -0.025 2010/01/01 06:00:00 $OUTDIR/warp.06 g $OUTDIR/diff.06 + 7 2 $OUTDIR/test.07 0.025 2010/01/01 07:00:00 $OUTDIR/warp.07 g $OUTDIR/diff.07 + 8 2 $OUTDIR/test.08 0.010 2010/01/01 08:00:00 $OUTDIR/warp.08 g $OUTDIR/diff.08 + 9 3 $OUTDIR/test.09 0.000 2010/01/01 09:00:00 $OUTDIR/warp.09 g $OUTDIR/diff.09 + 10 3 $OUTDIR/test.10 -0.025 2010/01/01 10:00:00 $OUTDIR/warp.10 g $OUTDIR/diff.10 + 11 3 $OUTDIR/test.11 0.025 2010/01/01 11:00:00 $OUTDIR/warp.11 g $OUTDIR/diff.11 + 12 3 $OUTDIR/test.12 0.010 2010/01/01 12:00:00 $OUTDIR/warp.12 g $OUTDIR/diff.12 + 13 4 $OUTDIR/test.13 -0.025 2010/01/02 01:00:00 $OUTDIR/warp.13 r $OUTDIR/diff.13 + 14 4 $OUTDIR/test.14 0.025 2010/01/02 02:00:00 $OUTDIR/warp.14 r $OUTDIR/diff.14 + 15 4 $OUTDIR/test.15 0.010 2010/01/02 03:00:00 $OUTDIR/warp.15 r $OUTDIR/diff.15 + 16 4 $OUTDIR/test.16 0.000 2010/01/02 04:00:00 $OUTDIR/warp.16 r $OUTDIR/diff.16 + 17 5 $OUTDIR/test.17 -0.025 2010/01/02 05:00:00 $OUTDIR/warp.17 r $OUTDIR/diff.17 + 18 5 $OUTDIR/test.18 0.025 2010/01/02 06:00:00 $OUTDIR/warp.18 r $OUTDIR/diff.18 + 19 5 $OUTDIR/test.19 0.010 2010/01/02 07:00:00 $OUTDIR/warp.19 r $OUTDIR/diff.19 + 20 5 $OUTDIR/test.20 0.000 2010/01/02 08:00:00 $OUTDIR/warp.20 r $OUTDIR/diff.20 + 21 6 $OUTDIR/test.21 -0.025 2010/01/02 09:00:00 $OUTDIR/warp.21 r $OUTDIR/diff.21 + 22 6 $OUTDIR/test.22 0.025 2010/01/02 10:00:00 $OUTDIR/warp.22 r $OUTDIR/diff.22 + 23 6 $OUTDIR/test.23 0.010 2010/01/02 11:00:00 $OUTDIR/warp.23 r $OUTDIR/diff.23 + 24 6 $OUTDIR/test.24 -0.020 2010/01/02 12:00:00 $OUTDIR/warp.24 r $OUTDIR/diff.24 end @@ -65,4 +65,5 @@ mkcatdir.stk PS1_V5 PS1_V5 mkcatdir.wrp PS1_V5_Lensing PS1_V5 + mkcatdir.dif PS1_DV5 PS1_V5 insert.stack.set end @@ -72,6 +73,8 @@ $dbhost = localhost $dbname = gpc1test - $dbuser = dvo - $dbpass = dvo +# $dbuser = dvo +# $dbpass = dvo + $dbuser = czw + $dbpass = czw exec mkgpc1.sh delete $dbhost $dbname $dbuser $dbpass @@ -82,5 +85,5 @@ # system-wide info dbinsert ippToPspsFake (my_id) values (2) - dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, $SKYCELL) + dbinsert skycell (radeg, decdeg, tess_id, skycell_id) values ($RA, $DEC, $TESS_ID, $SKYCELL) end @@ -477,4 +480,122 @@ end +macro mkcatdir.dif + if ($0 != 3) + echo "mkcatdir.cam (cmftype) (dvotype)" + break + end + + # tapPLAN 4 + + mkdir $OUTDIR + + local i catdir ID stkID rawfile cmffile offset + $catdir = $OUTDIR/catdir.dif + exec rm -rf $catdir + + $TIMEFORMAT = mjd + $TIMEREF = 2001/01/01,00:00:00 + + for i 0 $imagedata:n + list word -split $imagedata:$i + $ID = $word:0 + $stkID = $word:1 + $rawfile = $word:2\.txt + $cmffile = $word:8\.cmf + $offset = $word:3 + sprintf myDATE "%s" $word:4 + sprintf myTIME "%s" $word:5 + $filter = $word:7 + + if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue + + echo ctimes -abs $myDATE,$myTIME -var mjd + ctimes -abs $myDATE,$myTIME -var mjd + + # create an empty header and populate with the desired keywords + mcreate dummy 0 0 + keyword dummy MJD-OBS -wf $mjd + keyword dummy FILTERID -w $filter.00000 + keyword dummy EXPTIME -wf 1.0 + +# keyword dummy ZPT_ERR -wf 0.01 +# keyword dummy EXPREQ -wf 1.0 + keyword dummy AIRMASS -wf 1.0 + keyword dummy RA -wf $RA + keyword dummy DEC -wf $DEC + keyword dummy CTYPE1 -w "RA---DIS" + keyword dummy CTYPE2 -w "DEC--DIS" + keyword dummy CRVAL1 -wf $RA + keyword dummy CRVAL2 -wf $DEC + keyword dummy CRPIX1 -wf 0.0 + keyword dummy CRPIX2 -wf 0.0 + keyword dummy CDELT1 -wf {1.0/3600} + keyword dummy CDELT2 -wf {1.0/3600} + keyword dummy PC001001 -wf 1.0 + keyword dummy PC001002 -wf 0.0 + keyword dummy PC002001 -wf 0.0 + keyword dummy PC002002 -wf 1.0 + keyword dummy NPLYTERM -wd 0 + keyword dummy PCA1X3Y0 -wf 0.0 + keyword dummy PCA1X2Y1 -wf 0.0 + keyword dummy PCA1X1Y2 -wf 0.0 + keyword dummy PCA1X0Y3 -wf 0.0 + keyword dummy PCA1X2Y0 -wf 0.0 + keyword dummy PCA1X1Y1 -wf 0.0 + keyword dummy PCA1X0Y2 -wf 0.0 + keyword dummy PCA2X3Y0 -wf 0.0 + keyword dummy PCA2X2Y1 -wf 0.0 + keyword dummy PCA2X1Y2 -wf 0.0 + keyword dummy PCA2X0Y3 -wf 0.0 + keyword dummy PCA2X2Y0 -wf 0.0 + keyword dummy PCA2X1Y1 -wf 0.0 + keyword dummy PCA2X0Y2 -wf 0.0 + + $zp = $offset + 23.0 + keyword dummy "HIERARCH FPA.ZP" -wf $zp + keyword dummy "HIERARCH DETEFF.MAGREF" -wf -5.8048040 + wd dummy $cmffile + + mkinput $offset $rawfile + echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip -no-noise -extroot SkyChip -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC + exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip -no-noise -extroot SkyChip -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC + + + insert.dif $ID $ID $stkID $word:8 + + if ($i == 3) + echo "NOTE: not adding image $i to DVO : this simulates images with poor astrometry" + continue + end + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass + end + if ($SMALLTEST) + echo relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update + exec relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update + else + echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update + end + + if ($PARALLEL) + $hostname = `hostname` + exec rm -f $catdir/HostTable.dat + output $catdir/HostTable.dat + echo "# ID Hostname Catdir" + echo " 1 $hostname $catdir.p1" + echo " 2 $hostname $catdir.p2" + echo " 3 $hostname $catdir.p3" + output stdout + exec dvodist -out $catdir + end + + # for i 0 $offset:n + # tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i" + # end + + # tapDONE +end + macro insert.stack.set @@ -516,5 +637,42 @@ dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (3, 3, "full", "catdir.wrp") dbinsert mergedvodbProcessed (merge_id, fault) values (3, 0) -end + + # catdirs for the diffs + dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.dif", 4, "merged") + dbinsert minidvodbProcessed (minidvodb_id, fault) values (4, 0) + + dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (4, 4, "full", "catdir.dif") + dbinsert mergedvodbProcessed (merge_id, fault) values (4, 0) + +end + +macro insert.dif + if ($0 != 5) + echo "USAGE: insert.dif (diffID) (warpID) (stackID) (filename)" + break + end + + local difID wrpID stkID filename + $difID = $1 + $wrpID = $2 + $stkID = $3 + $filename = $4 + + echo $myDATE + echo $myTIME + + $addID = $difID + 100 + $diffSkyfileID = $difID + 25; + dbinsert addRun (add_id, stage, state, stage_id, stage_extra1, minidvodb_name) values ($addID, "diff", "full", $difID, $diffSkyfileID, "catdir.dif") + dbinsert addProcessedExp (add_id, fault) values ($addID, 0) + + dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, $TESS_ID, "full", 2, "38000M") + dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, $TESS_ID, $SKYCELL ) + dbinsert diffSkyfile (diff_id, diff_skyfile_id, tess_id, skycell_id, path_base) values ($difID, $diffSkyfileID, $TESS_ID, $SKYCELL, "$filename") +# dbinsert diffSkyfile (diff_id, + +# finish this up. +end +