Index: /branches/eam_branches/20100225/Ohana/src/relphot/src/StarOps.c =================================================================== --- /branches/eam_branches/20100225/Ohana/src/relphot/src/StarOps.c (revision 27525) +++ /branches/eam_branches/20100225/Ohana/src/relphot/src/StarOps.c (revision 27526) @@ -194,4 +194,5 @@ Nsecfilt = catalog[0].Nsecfilt; +# define PSFQUALSTATS 1 for (i = 0; i < Ncatalog; i++) { for (j = 0; j < catalog[i].Naverage; j++) { @@ -213,5 +214,5 @@ // XXX only apply this filter for psphot data from GPC1 for now... - if (0 && (catalog[i].measure[m].photcode > 10000) && (catalog[i].measure[m].photcode < 10500)) { + if (PSFQUALSTATS && (catalog[i].measure[m].photcode > 10000) && (catalog[i].measure[m].photcode < 10500)) { if (catalog[i].measure[m].psfQual < 0.85) continue; } @@ -235,5 +236,5 @@ // update average flags based on the detection stats. // XXX we need to clean this up -- this is a serious set of hacks... - if (0) { + if (PSFQUALSTATS) { int Galaxy2MASS, GalaxySDSS, goodPS1, nEXT, nPSF, good2MASS; Index: /branches/eam_branches/20100225/ippScripts/scripts/automate_stacks.pl =================================================================== --- /branches/eam_branches/20100225/ippScripts/scripts/automate_stacks.pl (revision 27525) +++ /branches/eam_branches/20100225/ippScripts/scripts/automate_stacks.pl (revision 27526) @@ -55,5 +55,5 @@ # Grab options -my ( $date, $camera, $dbname, $logfile, $verbose); +my ( $date, $camera, $dbname, $logfile, $verbose, $manual); my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only); my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); @@ -125,4 +125,5 @@ my %comment_list= (); my %stackable_list = (); +my %reduction_class = (); my $retention_time = 9000; @@ -168,4 +169,7 @@ $stackable_list{$this_target} = ${ $tentry }{value}; } + elsif (${ $tentry }{name} eq 'REDUCTION') { + $reduction_class{$this_target} = ${ $tentry }{value}; + } } } @@ -307,5 +311,5 @@ $Nsummit_exps++; if ($summit_fault) { - print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault"; + print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault"; if ($exp_type ne 'OBJECT') { print STDERR " (but I don't care).\n"; @@ -450,5 +454,5 @@ my $target = shift; - my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); my $select = "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; @@ -469,4 +473,7 @@ $cmd .= " -comment '$comment' "; } + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } $cmd .= " $select "; if ($debug == 1) { @@ -484,5 +491,5 @@ my $db = init_gpc_db(); $date =~ s/-//g; - my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id"; @@ -543,13 +550,13 @@ my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target); if ($Nexposures == 0) { - print STDERR "execute_chips: Target $target on $date had no exposures.\n"; + print STDERR "execute_chips: Target $target on $date had no exposures.\n"; + next; + } + if ($Nalready != 0) { + print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n"; next; } - if ($Nalready != 0) { - print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n"; - next; - } if ($Nimfiles != $Nburntooled) { - print STDERR "execute_chips: Target $target on $date is not fully burntooled.\n"; + print STDERR "execute_chips: Target $target on $date is not fully burntooled.\n"; $metadata_out{nsState} = 'NEEDSBURNING'; next; @@ -574,5 +581,5 @@ my $filter = shift; - my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; @@ -603,5 +610,5 @@ my $db = init_gpc_db(); $date =~ s/-//g; - my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'"; @@ -616,5 +623,5 @@ my $filter = shift; - my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); # check warp stage == chip stage my $db = init_gpc_db(); @@ -716,5 +723,5 @@ my $cleaning_date = $dt->ymd; - my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($cleaning_date,$target); + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); my $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -data_group $data_group "; @@ -779,6 +786,6 @@ my $data_group = "${target}.${trunc_date}"; my $tess_id = $tessID_list{$target}; - - return($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group); + my $reduction = $reduction_class{$target}; + return($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction); } Index: /branches/eam_branches/20100225/ippTasks/detrend.cleanup.pro =================================================================== --- /branches/eam_branches/20100225/ippTasks/detrend.cleanup.pro (revision 27525) +++ /branches/eam_branches/20100225/ippTasks/detrend.cleanup.pro (revision 27526) @@ -6,77 +6,39 @@ check.globals -book init detCleanupProcessedImfile -book init detCleanupProcessedExp +book init detCleanupProcessed +book init detCleanupResid + book init detCleanupStackedImfile book init detCleanupNormStatImfile book init detCleanupNormImfile book init detCleanupNormExp -book init detCleanupResidImfile -book init detCleanupResidExp + macro detclean.reset - book init detCleanupProcessedImfile - book init detCleanupProcessedExp + book init detCleanupProcessed + book init detCleanupResid + book init detCleanupStackedImfile - book init detCleanupNormStatImfile book init detCleanupNormImfile book init detCleanupNormExp - - book init detCleanupResidImfile - book init detCleanupResidExp end macro detclean.status - echo detCleanupProcessedImfile - book listbook detCleanupProcessedImfile - echo detCleanupProcessedExp - book listbook detCleanupProcessedExp - echo detCleanupStackedImfile + book listbook detCleanupProcessed + book listbook detCleanupResid + book listbook detCleanupStackedImfile - book listbook detCleanupNormStatImfile book listbook detCleanupNormImfile book listbook detCleanupNormExp - book listbook detCleanupResidImfile - book listbook detCleanupResidExp + end macro detclean.on - task detrend.cleanup.process.load + task detrend.cleanup.processed.load active true end - task detrend.cleanup.process.run - active true - end - task detrend.cleanup.processexp.load - active true - end - task detrend.cleanup.processexp.run - active true - end - task detrend.cleanup.stack.load - active true - end - task detrend.cleanup.stack.run - active true - end - - task detrend.cleanup.norm.load - active true - end - task detrend.cleanup.norm.run - active true - end - task detrend.cleanup.normexp.load - active true - end - task detrend.cleanup.normexp.run - active true - end - task detrend.cleanup.normstat.load - active true - end - task detrend.cleanup.normstat.run + task detrend.cleanup.processed.run active true end @@ -88,25 +50,46 @@ active true end - task detrend.cleanup.residexp.load - active true - end - task detrend.cleanup.residexp.run - active true - end + +# task detrend.cleanup.stack.load +# active true +# end +# task detrend.cleanup.stack.run +# active true +# end +# task detrend.cleanup.norm.load +# active true +# end +# task detrend.cleanup.norm.run +# active true +# end +# task detrend.cleanup.normexp.load +# active true +# end +# task detrend.cleanup.normexp.run +# active true +# end +# task detrend.cleanup.normstat.load +# active true +# end +# task detrend.cleanup.normstat.run +# active true +# end end macro detclean.off - task detrend.cleanup.process.load - active false - end - task detrend.cleanup.process.run - active false - end - task detrend.cleanup.processexp.load - active false - end - task detrend.cleanup.processexp.run - active false - end + task detrend.cleanup.processed.load + active false + end + task detrend.cleanup.processed.run + active false + end + + task detrend.cleanup.resid.load + active false + end + task detrend.cleanup.resid.run + active false + end + task detrend.cleanup.stack.load active false @@ -133,31 +116,19 @@ active false end - task detrend.cleanup.resid.load - active false - end - task detrend.cleanup.resid.run - active false - end - task detrend.cleanup.residexp.load - active false - end - task detrend.cleanup.residexp.run - active false - end + end # these variables will cycle through the known database names -$detCleanupProcessedImfile_DB = 0 -$detCleanupProcessedExp_DB = 0 +$detCleanupProcessed_DB = 0 +$detCleanupResid_DB = 0 + $detCleanupStackedImfile_DB = 0 $detCleanupNormStatImfile_DB = 0 $detCleanupNormImfile_DB = 0 $detCleanupNormExp_DB = 0 -$detCleanupResidImfile_DB = 0 -$detCleanupResidExp_DB = 0 - -######## cleanup process imfile ######## -task detrend.cleanup.process.load + +######## cleanup processed imfile/exp ######## +task detrend.cleanup.processed.load host local @@ -169,16 +140,16 @@ stdout NULL - stderr $LOGDIR/detrend.cleanup.process.imfile.log - - task.exec - $run = dettool -pendingcleanup_processedimfile + stderr $LOGDIR/detrend.cleanup.processed.log + + task.exec + $run = dettool -pendingcleanup_processedexp if ($DB:n == 0) option DEFAULT else # save the DB name for the exit tasks - option $DB:$detCleanupProcessedImfile_DB - $run = $run -dbname $DB:$detCleanupProcessedImfile_DB - $detCleanupProcessedImfile_DB ++ - if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0 + option $DB:$detCleanupProcessed_DB + $run = $run -dbname $DB:$detCleanupProcessed_DB + $detCleanupProcessed_DB ++ + if ($detCleanupProcessed_DB >= $DB:n) set detCleanupProcessed_DB = 0 end add_poll_args run @@ -189,11 +160,11 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout detCleanupProcessed -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) - book listbook detCleanupProcessedImfile + book listbook detCleanupProcessed end # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupProcessedImfile + process_cleanup detCleanupProcessed end @@ -214,5 +185,5 @@ # run the ipp_cleanup.pl script on pending images -task detrend.cleanup.process.run +task detrend.cleanup.processed.run periods -poll $RUNPOLL periods -exec $RUNEXEC @@ -221,28 +192,27 @@ task.exec - book npages detCleanupProcessedImfile -var N + book npages detCleanupProcessed -var N if ($N == 0) break if ($NETWORK == 0) break - # look for new images in detCleanupProcessedImfile (pantaskState == INIT) - book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT + # look for new images in detCleanupProcessed (pantaskState == INIT) + book getpage detCleanupProcessed 0 -var pageName -key pantaskState INIT if ("$pageName" == "NULL") break - book setword detCleanupProcessedImfile $pageName pantaskState RUN - book getword detCleanupProcessedImfile $pageName det_id -var DET_ID - book getword detCleanupProcessedImfile $pageName exp_id -var EXP_ID - book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID - book getword detCleanupProcessedImfile $pageName camera -var CAMERA - book getword detCleanupProcessedImfile $pageName data_state -var CLEANUP_MODE - book getword detCleanupProcessedImfile $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) + book setword detCleanupProcessed $pageName pantaskState RUN + book getword detCleanupProcessed $pageName det_id -var DET_ID + book getword detCleanupProcessed $pageName exp_id -var EXP_ID + book getword detCleanupProcessed $pageName camera -var CAMERA + book getword detCleanupProcessed $pageName data_state -var CLEANUP_MODE + book getword detCleanupProcessed $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and diff (class_id) set.host.for.camera $CAMERA FPA - stdout $LOGDIR/detrend.cleanup.process.imfile.log - stderr $LOGDIR/detrend.cleanup.process.imfile.log + stdout $LOGDIR/detrend.cleanup.processed.log + stderr $LOGDIR/detrend.cleanup.processed.log # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.processed --stage_id $DET_ID.$EXP_ID --camera $CAMERA --mode $CLEANUP_MODE --dbname $DBNAME add_standard_args run @@ -259,5 +229,5 @@ # default exit status task.exit default - process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS + process_exit detCleanupProcessed $options:0 $JOB_STATUS end @@ -266,5 +236,5 @@ showcommand crash echo "hostname: $JOB_HOSTNAME" - book setword detCleanupProcessedImfile $options:0 pantaskState CRASH + book setword detCleanupProcessed $options:0 pantaskState CRASH end @@ -272,11 +242,12 @@ task.exit timeout showcommand timeout - book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT - end -end - - -######## cleanup process exp ######## -task detrend.cleanup.processexp.load + book setword detCleanupProcessed $options:0 pantaskState TIMEOUT + end +end + + + +######## cleanup resid imfile/exp ######## +task detrend.cleanup.resid.load host local @@ -288,16 +259,16 @@ stdout NULL - stderr $LOGDIR/detrend.cleanup.process.exp.log - - task.exec - $run = dettool -pendingcleanup_processedexp + stderr $LOGDIR/detrend.cleanup.resid.log + + task.exec + $run = dettool -pendingcleanup_residexp if ($DB:n == 0) option DEFAULT else # save the DB name for the exit tasks - option $DB:$detCleanupProcessedExp_DB - $run = $run -dbname $DB:$detCleanupProcessedExp_DB - $detCleanupProcessedExp_DB ++ - if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0 + option $DB:$detCleanupResid_DB + $run = $run -dbname $DB:$detCleanupResid_DB + $detCleanupResid_DB ++ + if ($detCleanupResid_DB >= $DB:n) set detCleanupResid_DB = 0 end add_poll_args run @@ -308,11 +279,11 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout detCleanupResid -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) - book listbook detCleanupProcessedExp + book listbook detCleanupResid end # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupProcessedExp + process_cleanup detCleanupResid end @@ -333,5 +304,5 @@ # run the ipp_cleanup.pl script on pending images -task detrend.cleanup.processexp.run +task detrend.cleanup.resid.run periods -poll $RUNPOLL periods -exec $RUNEXEC @@ -340,27 +311,28 @@ task.exec - book npages detCleanupProcessedExp -var N + book npages detCleanupResid -var N if ($N == 0) break if ($NETWORK == 0) break - # look for new images in detCleanupProcessedExp (pantaskState == INIT) - book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT + # look for new images in detCleanupResid (pantaskState == INIT) + book getpage detCleanupResid 0 -var pageName -key pantaskState INIT if ("$pageName" == "NULL") break - book setword detCleanupProcessedExp $pageName pantaskState RUN - book getword detCleanupProcessedExp $pageName det_id -var DET_ID - book getword detCleanupProcessedExp $pageName exp_id -var EXP_ID - book getword detCleanupProcessedExp $pageName camera -var CAMERA - book getword detCleanupProcessedExp $pageName data_state -var CLEANUP_MODE - book getword detCleanupProcessedExp $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) + book setword detCleanupResid $pageName pantaskState RUN + book getword detCleanupResid $pageName det_id -var DET_ID + book getword detCleanupResid $pageName exp_id -var EXP_ID + book getword detCleanupResid $pageName iteration -var ITERATION + book getword detCleanupResid $pageName camera -var CAMERA + book getword detCleanupResid $pageName data_state -var CLEANUP_MODE + book getword detCleanupResid $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and diff (class_id) set.host.for.camera $CAMERA FPA - stdout $LOGDIR/detrend.cleanup.process.exp.log - stderr $LOGDIR/detrend.cleanup.process.exp.log + stdout $LOGDIR/detrend.cleanup.resid.log + stderr $LOGDIR/detrend.cleanup.resid.log # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage detrend.resid --stage_id $DET_ID.$EXP_ID --camera $CAMERA --mode $CLEANUP_MODE --dbname $DBNAME add_standard_args run @@ -377,5 +349,5 @@ # default exit status task.exit default - process_exit detCleanupProcessedExp $options:0 $JOB_STATUS + process_exit detCleanupResid $options:0 $JOB_STATUS end @@ -384,5 +356,5 @@ showcommand crash echo "hostname: $JOB_HOSTNAME" - book setword detCleanupProcessedExp $options:0 pantaskState CRASH + book setword detCleanupResid $options:0 pantaskState CRASH end @@ -390,5 +362,5 @@ task.exit timeout showcommand timeout - book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT + book setword detCleanupResid $options:0 pantaskState TIMEOUT end end @@ -402,5 +374,5 @@ periods -timeout 30 npending 1 - active true + active false stdout NULL @@ -449,4 +421,5 @@ end + # run the ipp_cleanup.pl script on pending images task detrend.cleanup.stack.run @@ -521,5 +494,5 @@ periods -timeout 30 npending 1 - active true + active false stdout NULL @@ -573,5 +546,5 @@ periods -exec $RUNEXEC periods -timeout 60 - active true + active false task.exec @@ -636,5 +609,5 @@ periods -timeout 30 npending 1 - active true + active false stdout NULL @@ -688,5 +661,5 @@ periods -exec $RUNEXEC periods -timeout 60 - active true + active false task.exec @@ -752,5 +725,5 @@ periods -timeout 30 npending 1 - active true + active false stdout NULL @@ -804,5 +777,5 @@ periods -exec $RUNEXEC periods -timeout 60 - active true + active false task.exec @@ -860,239 +833,2 @@ -######## cleanup resid imfile ######## -task detrend.cleanup.resid.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.resid.imfile.log - - task.exec - $run = dettool -pendingcleanup_residimfile - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupResidImfile_DB - $run = $run -dbname $DB:$detCleanupResidImfile_DB - $detCleanupResidImfile_DB ++ - if ($detCleanupResidImfile_DB >= $DB:n) set detCleanupResidImfile_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupResidImfile - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupResidImfile - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.resid.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupResidImfile -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupResidImfile (pantaskState == INIT) - book getpage detCleanupResidImfile 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupResidImfile $pageName pantaskState RUN - book getword detCleanupResidImfile $pageName det_id -var DET_ID - book getword detCleanupResidImfile $pageName exp_id -var EXP_ID - book getword detCleanupResidImfile $pageName class_id -var CLASS_ID - book getword detCleanupResidImfile $pageName iteration -var ITERATION - book getword detCleanupResidImfile $pageName camera -var CAMERA - book getword detCleanupResidImfile $pageName data_state -var CLEANUP_MODE - book getword detCleanupResidImfile $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.resid.imfile.log - stderr $LOGDIR/detrend.cleanup.resid.imfile.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupResidImfile $options:0 $JOB_STATUS - end - - # locked list - task.exit crash - showcommand crash - echo "hostname: $JOB_HOSTNAME" - book setword detCleanupResidImfile $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupResidImfile $options:0 pantaskState TIMEOUT - end -end - - -######## cleanup resid exp ######## -task detrend.cleanup.residexp.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.resid.exp.log - - task.exec - $run = dettool -pendingcleanup_residexp - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupResidExp_DB - $run = $run -dbname $DB:$detCleanupResidExp_DB - $detCleanupResidExp_DB ++ - if ($detCleanupResidExp_DB >= $DB:n) set detCleanupResidExp_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupResidExp - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupResidExp - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.residexp.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupResidExp -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupResidExp (pantaskState == INIT) - book getpage detCleanupResidExp 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupResidExp $pageName pantaskState RUN - book getword detCleanupResidExp $pageName det_id -var DET_ID - book getword detCleanupResidExp $pageName exp_id -var EXP_ID - book getword detCleanupResidExp $pageName iteration -var ITERATION - book getword detCleanupResidExp $pageName camera -var CAMERA - book getword detCleanupResidExp $pageName data_state -var CLEANUP_MODE - book getword detCleanupResidExp $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.resid.exp.log - stderr $LOGDIR/detrend.cleanup.resid.exp.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupResidExp $options:0 $JOB_STATUS - end - - # locked list - task.exit crash - showcommand crash - echo "hostname: $JOB_HOSTNAME" - book setword detCleanupResidExp $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupResidExp $options:0 pantaskState TIMEOUT - end -end Index: /branches/eam_branches/20100225/ippTasks/nightly_stacks.pro =================================================================== --- /branches/eam_branches/20100225/ippTasks/nightly_stacks.pro (revision 27525) +++ /branches/eam_branches/20100225/ippTasks/nightly_stacks.pro (revision 27526) @@ -605,5 +605,5 @@ host $host # set.host.for.camera GPC1 $class_id - $logfile = "burntool_logs/$class_id.$start_date.log" + $logfile = "/data/${host}.0/burntool_logs/$class_id.$start_date.log" $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile Index: /branches/eam_branches/20100225/ippToPsps/scripts/pspsSchema2xml.pl =================================================================== --- /branches/eam_branches/20100225/ippToPsps/scripts/pspsSchema2xml.pl (revision 27525) +++ /branches/eam_branches/20100225/ippToPsps/scripts/pspsSchema2xml.pl (revision 27526) @@ -89,5 +89,5 @@ while () { - if ($_ =~ m/.*CREATE TABLE.*$tableName/) { + if ($_ =~ m/.*CREATE TABLE\s+dbo\.$tableName\s*\(/i) { close (FILE); @@ -209,5 +209,5 @@ } - if($reading && $_ =~ m/^\)/) {$reading = 0;} + if($reading && $_ =~ m/^\s*\)\s*/) {$reading = 0;} if(!$reading) {next;} @@ -218,5 +218,5 @@ if (!$line) {next;} if (length($line) < 5) {next;} - if ($line =~ m/^--/) {next;} + if ($line =~ m/^\s*--/) {next;} if ($line =~ m/\/\*/) {next;} if ($line =~ m/\*\//) {next;} Index: /branches/eam_branches/20100225/ippTools/share/magicdstool_getskycells.sql =================================================================== --- /branches/eam_branches/20100225/ippTools/share/magicdstool_getskycells.sql (revision 27525) +++ /branches/eam_branches/20100225/ippTools/share/magicdstool_getskycells.sql (revision 27526) @@ -1,5 +1,6 @@ +-- Input warps SELECT DISTINCT diffSkyfile.diff_id, - diffRun.tess_id, + diffRun.tess_id, diffSkyfile.skycell_id, diffSkyfile.path_base, @@ -8,5 +9,5 @@ JOIN magicRun USING(magic_id) JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun ON magicRun.diff_id = diffRun.diff_id +JOIN diffRun USING(diff_id) JOIN diffSkyfile ON magicRun.diff_id = diffSkyfile.diff_id @@ -17,4 +18,5 @@ -- Want input warps only AND diffInputSkyfile.warp1 IS NOT NULL + AND magicRun.inverse = 0 JOIN warpSkyCellMap ON warpSkyCellMap.warp_id = diffInputSkyfile.warp1 @@ -27,3 +29,34 @@ diffSkyfile.fault = 0 AND diffSkyfile.quality = 0 - AND magic_ds_id = %lld + -- WHERE hook %s +UNION +-- Reference warps +SELECT DISTINCT + diffSkyfile.diff_id, + diffRun.tess_id, + diffSkyfile.skycell_id, + diffSkyfile.path_base, + diffSkyfile.data_state +FROM magicDSRun +JOIN magicRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffSkyfile.diff_id + AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id + AND diffInputSkyfile.warp2 IS NOT NULL + AND magicRun.inverse = 1 +JOIN warpSkyCellMap + ON warpSkyCellMap.warp_id = diffInputSkyfile.warp2 + AND warpSkyCellMap.skycell_id = diffInputSkyfile.skycell_id +JOIN warpSkyfile + ON warpSkyfile.warp_id = warpSkyCellMap.warp_id + AND warpSkyfile.skycell_id = warpSkyCellMap.skycell_id + AND warpSkyfile.quality = 0 +WHERE + diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + -- WHERE hook %s Index: /branches/eam_branches/20100225/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/20100225/ippTools/src/magicdstool.c (revision 27525) +++ /branches/eam_branches/20100225/ippTools/src/magicdstool.c (revision 27526) @@ -1135,4 +1135,5 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); @@ -1146,12 +1147,12 @@ } + psString whereClause = NULL; if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause, "\n AND "); } psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); Index: /branches/eam_branches/20100225/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/eam_branches/20100225/ippconfig/recipes/nightly_science.config (revision 27525) +++ /branches/eam_branches/20100225/ippconfig/recipes/nightly_science.config (revision 27526) @@ -119,20 +119,153 @@ STACKABLE BOOL FALSE END -TARGETS METADATA - NAME STR microtest +# This is a hack, and I freely admit it. +TARGETS METADATA + NAME STR microtestMD01 + TESS STR MD01 + OBJECT STR MD01% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD01.noPattern + TESS STR MD01 + OBJECT STR MD01% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD02 + TESS STR MD02 + OBJECT STR MD02% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD02.noPattern + TESS STR MD02 + OBJECT STR MD02% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD03 + TESS STR MD03 + OBJECT STR MD03% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD03.noPattern + TESS STR MD03 + OBJECT STR MD03% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD04 TESS STR MD04 - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -# TARGETS METADATA -# NAME STR CMB -# TESS STR RINGS.V0 -# COMMENT STR CMB_Cold% -# STACKABLE BOOL FALSE -# END -# TARGETS METADATA -# NAME STR SVS -# TESS STR RINGS.V0 -# COMMENT STR SVS% -# STACKABLE BOOL FALSE -# END + OBJECT STR MD04% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD04.noPattern + TESS STR MD04 + OBJECT STR MD04% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD05 + TESS STR MD05 + OBJECT STR MD05% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD05.noPattern + TESS STR MD05 + OBJECT STR MD05% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD06 + TESS STR MD06 + OBJECT STR MD06% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD06.noPattern + TESS STR MD06 + OBJECT STR MD06% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD07 + TESS STR MD07 + OBJECT STR MD07% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD07.noPattern + TESS STR MD07 + OBJECT STR MD07% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD08 + TESS STR MD08 + OBJECT STR MD08% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD08.noPattern + TESS STR MD08 + OBJECT STR MD08% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD09 + TESS STR MD09 + OBJECT STR MD09% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD09.noPattern + TESS STR MD09 + OBJECT STR MD09% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD10 + TESS STR MD10 + OBJECT STR MD10% + COMMENT STR Microtest Obs% + STACKABLE BOOL TRUE +END +TARGETS METADATA + NAME STR microtestMD10.noPattern + TESS STR MD10 + OBJECT STR MD10% + COMMENT STR Microtest Obs% + REDUCTION STR STDSCIENCE.V0 + STACKABLE BOOL TRUE +END Index: /branches/eam_branches/20100225/ippconfig/recipes/psphot.config =================================================================== --- /branches/eam_branches/20100225/ippconfig/recipes/psphot.config (revision 27525) +++ /branches/eam_branches/20100225/ippconfig/recipes/psphot.config (revision 27526) @@ -70,4 +70,7 @@ MOMENTS_AR_MAX F32 1.5 # maximum axial ratio: 1 / AR < (sx / sy) < AR MOMENTS_GAUSS_SIGMA F32 4.0 # XXX this is now autoscaled + +# calculate difference stats? +DIFF_STATS BOOL FALSE # basic object statistics @@ -264,5 +267,5 @@ PSPHOT.CR.NSIGMA.LIMIT F32 3.0 # sources with crNsigma greater that this get tagged as likely cosmic rays PSPHOT.EXT.NSIGMA.LIMIT F32 3.0 # sources with extNsigma greater that this get tagged as likely extended sources -PSPHOT.EXT.NSIGMA.MOMENTS F32 2.0 # sources with extNsigma greater that this get tagged as likely extended sources +PSPHOT.EXT.NSIGMA.MOMENTS F32 3.0 # sources with extNsigma greater that this get tagged as likely extended sources PSPHOT.CR.GROW S32 1 # Number of pixels to grow CR mask PSPHOT.CR.NSIGMA.SOFTEN F32 0.0025 # Softening parameter for weights Index: /branches/eam_branches/20100225/pstamp/scripts/pstampparse.pl =================================================================== --- /branches/eam_branches/20100225/pstamp/scripts/pstampparse.pl (revision 27525) +++ /branches/eam_branches/20100225/pstamp/scripts/pstampparse.pl (revision 27526) @@ -233,4 +233,11 @@ # user requested us to search all components. Set to "" $search_component = "" if $search_component eq "all"; + + if (!check_image_type($stage)) { + print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose; + insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); + $num_jobs++; + next; + } if ((($job_type eq "stamp") or ($req_type eq "bycoord")) and ! validROI($row)) { @@ -958,4 +965,20 @@ } +sub check_image_type +{ + my $img_type = shift; + if (!$img_type) { + print STDERR "NULL IMG_TYPE supplied\n"; + return 0; + } + if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or + ($img_type eq "stack") or ($img_type eq "diff")) { + return 1; + } else { + print STDERR "$img_type is not a valid IMG_TYPE\n"; + return 0; + } +} + sub my_die { Index: /branches/eam_branches/20100225/tools/neb-ds9.pl =================================================================== --- /branches/eam_branches/20100225/tools/neb-ds9.pl (revision 27525) +++ /branches/eam_branches/20100225/tools/neb-ds9.pl (revision 27526) @@ -2,10 +2,18 @@ use Getopt::Std; -getopts('AM',\%opt); +use File::Temp qw( tempfile ); +getopts('AMDRC',\%opt); $cmd = 'ds9 -scale mode zscale '; - +if (exists($opt{M})) { + $cmd .= ' -mosaic wcs '; +} if (exists($opt{A})) { $cmd .= " -wcs align yes "; +} + +if (exists($opt{C})) { + $cmd .= " -frame new rgb -rgb system image "; + @color = ("-red","-blue","-green"); } @@ -16,9 +24,37 @@ $cmd .= "-medatacube $real_file "; } + elsif (($real_file =~ /cmf$/)) { + my $head_name = $real_file; + $head_name =~ s/.*XY(\d\d).*/XY$1.psf/; + my ($tempFile, $tempName) = tempfile("/tmp/ds9.$$.XXXX", + UNLINK => 1, SUFFIX => '.reg' ); + + if (exists($opt{R})) { + system("fdump $real_file $head_name | awk '{print(\"FK5;point(\",\$15,\$16,\") # point=boxcircle text = {\",\$8,\"}\"}' > $tempName"); + $cmd .= " -regions format xy -regions $tempName "; + } + else { + system("echo '# Region file format: DS9 version 4.1' > $tempName"); + system("echo 'global color=green dashlist=8 3 width=1 font=\"helvetica 10 normal\" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1' >> $tempName"); + + system("fdump $real_file $head_name | awk '{print(\"image;ellipse(\",\$2,\$3,\$22,\$23,\$24,\") # text = {\",\$8,\"\}\")}' >> $tempName"); + $cmd .= " -regions format ds9 -regions $tempName "; + } + + + } else { + if (exists($opt{C})) { + if ($#color == -1) { + next; + } + else { + $c = shift(@color); + $cmd .= " $c "; + } + } $cmd .= " $real_file "; } } } -#$cmd .= " -zoom to fit -regions format xy -regions system wcs -regions load targets.reg "; system("$cmd"); Index: /branches/eam_branches/20100225/tools/wiki_nightly_stacks_table.pl =================================================================== --- /branches/eam_branches/20100225/tools/wiki_nightly_stacks_table.pl (revision 27525) +++ /branches/eam_branches/20100225/tools/wiki_nightly_stacks_table.pl (revision 27526) @@ -22,10 +22,14 @@ $db = init_gpc_db(); -@targets = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10'); +@targets = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10','ThreePi','STS','SweetSpot'); # potential targets if (exists($opt{P})) { foreach $t (@targets) { - $sth = "select filter,DATE_FORMAT(rawExp.dateobs,'%Y-%m-%d') AS day,count(filter) FROM rawExp JOIN newExp using(exp_id) WHERE rawExp.fault = 0 AND exp_type = 'OBJECT' AND comment LIKE '${t}%' AND dateobs >= '2009-12-08T00:00:00' GROUP BY day,filter"; + $comment = $t; + if ($comment eq 'ThreePi') { + $comment = '3Pi'; + } + $sth = "select filter,DATE_FORMAT(rawExp.dateobs,'%Y-%m-%d') AS day,count(filter) FROM rawExp JOIN newExp using(exp_id) WHERE rawExp.fault = 0 AND exp_type = 'OBJECT' AND comment LIKE '%${comment}%' AND dateobs >= '2010-02-12T00:00:00' GROUP BY day,filter"; $data_ref = $db->selectall_arrayref( $sth ); foreach $rr (@{ $data_ref }) { @@ -63,5 +67,5 @@ #queued in chip -$sth = "select rawExp.dateobs,chipRun.state,rawExp.filter,count(chipRun.state),chipRun.data_group,chipRun.dist_group from chipRun JOIN rawExp using (exp_id) where chipRun.label LIKE '%nightlystack' GROUP BY data_group,filter,state ORDER BY dateobs"; +$sth = "select rawExp.dateobs,chipRun.state,rawExp.filter,count(chipRun.state),chipRun.data_group,chipRun.dist_group from chipRun JOIN rawExp using (exp_id) where chipRun.label LIKE '%nightlyscience' GROUP BY data_group,filter,state ORDER BY dateobs"; $data_ref = $db->selectall_arrayref( $sth ); @@ -133,5 +137,5 @@ #warps -$sth = "select rawExp.dateobs,warpRun.state,rawExp.filter,count(warpRun.state),warpRun.data_group,warpRun.dist_group from warpRun JOIN fakeRun using (fake_id) JOIN camRun using (cam_id) JOIN chipRun using (chip_id) JOIN rawExp using (exp_id) where warpRun.label LIKE '%nightlystack' GROUP BY data_group,filter,state ORDER BY data_group,filter"; +$sth = "select rawExp.dateobs,warpRun.state,rawExp.filter,count(warpRun.state),warpRun.data_group,warpRun.dist_group from warpRun JOIN fakeRun using (fake_id) JOIN camRun using (cam_id) JOIN chipRun using (chip_id) JOIN rawExp using (exp_id) where warpRun.label LIKE '%nightlyscience' GROUP BY data_group,filter,state ORDER BY data_group,filter"; $data_ref = $db->selectall_arrayref( $sth ); @@ -175,5 +179,5 @@ -$sth = "select state,count(skycell_id),data_group,dist_group,filter from stackRun where label LIKE '%nightlystack' group by data_group,filter,state"; +$sth = "select state,count(skycell_id),data_group,dist_group,filter from stackRun where label LIKE '%nightlyscience' group by data_group,filter,state"; $data_ref = $db->selectall_arrayref( $sth ); @@ -245,5 +249,5 @@ # distribution -$sth = "select distRun.data_group,dist_group,filter,count(stage_id) from distRun JOIN stackRun ON (stage_id = stack_id) where distRun.label LIKE '%.nightlystack' AND distRun.state = 'full' GROUP BY distRun.data_group,filter"; +$sth = "select distRun.data_group,dist_group,filter,count(stage_id) from distRun JOIN stackRun ON (stage_id = stack_id) where distRun.label LIKE '%.nightlyscience' AND distRun.state = 'full' GROUP BY distRun.data_group,filter"; $data_ref = $db->selectall_arrayref( $sth ); @@ -284,5 +288,5 @@ # faulted distributed -$sth = "select * from rcDSFileset join distRun using(dist_id) where rcDSFileset.fault > 0 and date(registered) >= '2010-01-01' AND label LIKE 'M%.nightlystack' order by fs_id asc"; +$sth = "select * from rcDSFileset join distRun using(dist_id) where rcDSFileset.fault > 0 and date(registered) >= '2010-01-01' AND label LIKE 'M%.nightlyscience' order by fs_id asc"; $data_ref = $db->selectall_arrayref( $sth );