Index: trunk/ippTasks/calibration.pro =================================================================== --- trunk/ippTasks/calibration.pro (revision 15313) +++ trunk/ippTasks/calibration.pro (revision 15313) @@ -0,0 +1,181 @@ +## this file contains the tasks for running the calibration stage +## these tasks use the books XXX + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/calibration +exec mkdir -p $LOGSUBDIR + +book init calBook + +macro calibration.reset + book init calBook +end + +macro calibration.status + book listbook calBook +end + +macro calibration.on + task calibration.init + active true + end +end + +macro calibration.off + task calibration.init + active false + end +end + +# these variables will cycle through the known database names +$calInit_DB = 0 + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.init + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 60 + trange Hourly@00:00 Hourly@00:10 + # every hour on the hour + + if ($DB:n == 0) + npending $DB:n + else + npending 1 + end + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + ## XXX have caltool define the regions, or have the task.exit below do it? + + # generate queues for all of the known (and active) DVO databases + task.exec + # get the current ST: + $RAs = 15*($ST - 1) + $RAe = 15*($ST - 2) + $DECs = -90.0 + $DECe = +90.0 + + $REGION = "$RAs,$RAe:$DECs,$DECe" + + if ($DB:n == 0) + option DEFAULT + command caltool -databases -active true + else + # save the DB name for the exit tasks + option $DB:$calInit_DB + option $REGION + command caltool -databases -dbname $DB:$calInit_DB -region $REGION + $calInit_DB ++ + if ($calInit_DB >= $DB:n) set calInit_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout calBook -key dbname:dvo_id -uniq -setword dbname $options:0 -setword region $options:1 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook calBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup calBook + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.resort + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 60 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + task.exec + book npages calBook -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in calBook + # the sequencing in this task set is by the pantasksState (see notes.txt) + book getpage calBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword calBook $pageName pantaskState RUN + + # XXX probably need to set the output / log based on WORKDIR... + book getword calBook $pageName catdir -var CATDIR + book getword calBook $pageName region -var REGION + book getword calBook $pageName dbname -var DBNAME + + # specify choice of remote host + if ($parallel) + host anyhost + else + host local + end + + ## XXX generate output log based on filerule? + ## XXX put this in a function? + + # save the pageName for future reference below + options $pageName + + # XXX do we modify relphot to loop over the filters? + $run = calibrate_dvo.pl --catdir $CATDIR --region $REGION + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit calBook $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword calBook $options:0 pantaskState TIMEOUT + end +end Index: trunk/ippTasks/detrend.correct.pro =================================================================== --- trunk/ippTasks/detrend.correct.pro (revision 15313) +++ trunk/ippTasks/detrend.correct.pro (revision 15313) @@ -0,0 +1,331 @@ +## this file contains the tasks for running the detrend correction stage +## these tasks use the books detPendingCorrectedImfile and detPendingCorrectedExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingCorrectedImfile +book init detPendingCorrectedExp + +macro detcorr.reset + book init detPendingCorrectedImfile + book init detPendingCorrectedExp +end + +macro detcorr.status + echo detPendingCorrectedImfile + book listbook detPendingCorrectedImfile + echo detPendingCorrectedExp + book listbook detPendingCorrectedExp +end + +macro detcorr.on + task detrend.correct.load + active true + end + task detrend.correct.run + active true + end + task detrend.correctexp.load + active true + end + task detrend.correctexp.run + active true + end +end + +macro detcorr.off + task detrend.correct.load + active false + end + task detrend.correct.run + active false + end + task detrend.correctexp.load + active false + end + task detrend.correctexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingCorrectedImfile_DB = 0 +$detPendingCorrectedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingCorrectedImfile +# compare the new list with the ones already selected +task detrend.correct.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detcorr.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tocorrectedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedImfile_DB + command dettool -tocorrectedimfile -limit 20 -dbname $DB:$detPendingCorrectedImfile_DB + $detPendingCorrectedImfile_DB ++ + if ($detPendingCorrectedImfile_DB >= $DB:n) set detPendingCorrectedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_correct_imfile.pl on pending images +task detrend.correct.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingCorrectedImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingCorrectedImfile + book getpage detPendingCorrectedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedImfile $pageName pantaskState RUN + book getword detPendingCorrectedImfile $pageName det_id -var DET_ID + book getword detPendingCorrectedImfile $pageName exp_id -var EXP_ID + book getword detPendingCorrectedImfile $pageName class_id -var CLASS_ID + book getword detPendingCorrectedImfile $pageName exp_tag -var EXP_TAG XXX where does this come from + book getword detPendingCorrectedImfile $pageName det_type -var DET_TYPE XXX where does this come from + book getword detPendingCorrectedImfile $pageName camera -var CAMERA XXX where does this come from + book getword detPendingCorrectedImfile $pageName uri -var URI + book getword detPendingCorrectedImfile $pageName corr_id -var CORR_ID + book getword detPendingCorrectedImfile $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedImfile $pageName workdir -var WORKDIR + book getword detPendingCorrectedImfile $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + ## output log filename + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detcorr.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_correct_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingCorrectedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingCorrectedExp +# compare the new list with the ones already selected +task detrend.correctexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detcorr.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tocorrectedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedExp_DB + command dettool -tocorrectedexp -limit 20 -dbname $DB:$detPendingCorrectedExp_DB + $detPendingCorrectedExp_DB ++ + if ($detPendingCorrectedExp_DB >= $DB:n) set detPendingCorrectedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.correctexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingCorrectedExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new exposures in detPendingCorrectedExp + book getpage detPendingCorrectedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedExp $pageName pantaskState RUN + book getword detPendingCorrectedExp $pageName det_id -var DET_ID + book getword detPendingCorrectedExp $pageName exp_id -var EXP_ID + book getword detPendingCorrectedExp $pageName det_type -var DET_TYPE + book getword detPendingCorrectedExp $pageName camera -var CAMERA + book getword detPendingCorrectedExp $pageName exp_tag -var EXP_TAG + book getword detPendingCorrectedExp $pageName corr_id -var CORR_ID + book getword detPendingCorrectedExp $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedExp $pageName workdir -var WORKDIR + book getword detPendingCorrectedExp $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # XXX push this into the filerules? + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detcorr.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + $logfile = $outroot.log + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_correct_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingCorrectedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedExp $options:0 pantaskState TIMEOUT + end +end Index: trunk/ippTasks/notes.txt =================================================================== --- trunk/ippTasks/notes.txt (revision 15298) +++ trunk/ippTasks/notes.txt (revision 15313) @@ -1,2 +1,38 @@ + +2007.10.11 + + The calibration suite uses a database table of known/registered DVO + databases to be calibrated. The suite operates on a segment of the + sky in each run, nominally a section ~1h west of the current ST. + + caltool -pendingdbs returns a list of entries (mdc format), one for + each DVO database in the table. + + caltool -databases [-active] : returns the list of dvo databases + caltool -addcalrun : inserts the stats for a single calibration run + caltool -setstate (active/inactive) : modify the state of a dvo + database + caltool -adddatabase : insert a new database + + XXX a potential source of collisions. the books are marked by + unique keys in the db tables, but a single pantasks run may be + addressing multiple databases. IDs are not unique across + databases. we can solve this by including in the key the dbname + + the calibration suite runs the following tasks, in sequences on each + of the databases: + + * resort, relphot, uniphot, relastro.objects, relastro.images + + ** detrend correction: any detrend images which need to be corrected + get inserted in the detCorrectedExp table. + + dettool commands needed: + * add a detrend exposure (det_id, exp_id) to the detCorrectedExp table + * add a detrend imfile (det_id, exp_id, class_id) to the + detCorrectedImfile table + * select pending detCorrectedExp + * select pending detCorrectedImfiles + 2007.01.08