Index: trunk/ippScripts/scripts/automate_stacks.pl =================================================================== --- trunk/ippScripts/scripts/automate_stacks.pl (revision 28221) +++ trunk/ippScripts/scripts/automate_stacks.pl (revision 28222) @@ -20,4 +20,5 @@ my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); @@ -59,4 +60,5 @@ my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); +my ( $check_detrends, $queue_detrends); GetOptions( @@ -81,4 +83,6 @@ 'check_stacks' => \$check_stacks, 'queue_stacks' => \$queue_stacks, + 'check_detrends' => \$check_detrends, + 'queue_detrends' => \$queue_detrends, 'check_diffs' => \$check_diffs, 'queue_diffs' => \$queue_diffs, @@ -105,4 +109,6 @@ --check_stacks Confirm that stacks can be built. --queue_stacks Issue stacktool commands to queue stacks. + --check_detrends Confirm that detrend verify runs can be built. + --queue_detrends Issue dettool commands to queue detrend verify runs. --check_diffs Confirm that diffs can be done. --queue_diffs Issue difftool commands to queue diffs. @@ -115,6 +121,6 @@ ) unless defined $check_registration or defined $define_burntool or defined $queue_burntool or - defined $queue_chips or defined $queue_stacks or - defined $check_chips or defined $check_stacks or + defined $queue_chips or defined $queue_stacks or $queue_detrends or + defined $check_chips or defined $check_stacks or $check_detrends or defined $test_mode or defined $clean_old or defined $check_mode; @@ -128,4 +134,11 @@ my %stackable_list = (); my %reduction_class = (); +my @detrend_list = (); +my %dettype_list = (); +my %exptype_list = (); +my %refID_list = (); +my %refIter_list = (); +my %detfilter_list = (); +my %detmax_list = (); my %clean_commands = (); my %clean_retention = (); @@ -158,7 +171,4 @@ } } -# if (${ $entry }{name} eq 'RETENTION_TIME') { -# $retention_time = ${ $entry }{value}; -# } elsif (${ $entry }{name} eq 'FILTERS') { push @filter_list, ${ $entry }{value}; @@ -195,8 +205,33 @@ } } -} - - - + elsif (${ $entry }{name} eq 'DETRENDS') { + my @detrend_data = @{ ${ $entry }{value} }; + my $this_detrend = ''; + foreach my $dentry (@detrend_data) { + if (${ $dentry }{name} eq 'NAME') { + $this_detrend = ${ $dentry }{value}; + push @detrend_list, $this_detrend; + } + elsif (${ $dentry }{name} eq 'DETTYPE') { + $dettype_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'EXPTYPE') { + $exptype_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'REF_ID') { + $refID_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'REF_ITER') { + $refIter_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'FILTER') { + $detfilter_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'MAX') { + $detmax_list{$this_detrend} = ${ $dentry }{value}; + } + } + } +} unless(defined($date)) { @@ -256,4 +291,16 @@ return_metadata($date); unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($check_detrends) || defined($test_mode)) { + $metadata_out{nsState} = 'CHECKDETRENDS'; + &execute_detrends($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_detrends)) { + $metadata_out{nsState} = 'QUEUEDETRENDS'; + &execute_detrends($date,"pretend"); + return_metadata($date); + exit(0); } if (defined($define_burntool) || defined($test_mode)) { @@ -498,5 +545,5 @@ my $cmd = "$chiptool"; - $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp '; + $cmd .= " -simple -dbname $dbname -definebyquery -set_end_stage warp "; $cmd .= " -set_label $label "; $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; @@ -566,4 +613,6 @@ } + + sub chip_queue { my $date = shift; @@ -612,4 +661,101 @@ # +# Detrend verification +################################################################################ + +sub construct_dettool_cmd { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + + my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$dettool"; + $cmd .= " -pretend "; + $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type "; + $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter "; + $cmd .= " $select "; + $cmd .= " -inst $camera "; + $cmd .= " -select_exp_type $exp_type "; + $cmd .= " -select_filter $filter " if defined($filter); + $cmd .= " -workdir $workdir "; + $cmd .= " -label $label "; + if ($maxN > 0) { + $cmd .= " -random_subset -random_limit $maxN "; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_detverify { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + + my $db = init_gpc_db(); + my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub pre_detrend_queue { + my $date = shift; + my $target = shift; + + my $command = construct_dettool_cmd($date,$target) . ' -pretend '; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + return($#input_exposures + 1); +} + +sub detrend_queue { + my $date = shift; + my $target = shift; + + my $command = construct_dettool_cmd($date,$target); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $metadata_out{nsState} = 'DETREND_QUEUED'; + return(0); +} + +sub execute_detrends { + my $date = shift; + my $pretend = shift; + my $exposures = 0; + foreach my $target (@detrend_list) { + my ($Nexposures) = pre_detrend_queue($date,$target); + if ($Nexposures == 0) { + print STDERR "execute_detrends: Target $target on $date had no exposures.\n"; + next; + } + $exposures++; + unless(defined($pretend)) { + detrend_queue($date,$target); + } + } + if ($exposures == 0) { + $metadata_out{nsState} = 'DETREND_DROP'; + } +} + +# # Stacking ################################################################################ @@ -627,5 +773,5 @@ my $cmd = "$stacktool"; # $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 '; # Probably silly, but I want to be safe and not overwrite - $cmd .= ' -simple -dbname gpc1 -definebyquery '; + $cmd .= " -simple -dbname $dbname -definebyquery "; $cmd .= " -set_label $label -select_label $label "; $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; @@ -779,5 +925,5 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); my $args = $command; - $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; if ($debug == 1) { $args .= ' -pretend '; @@ -826,5 +972,5 @@ my $label = "${target}.nightlyscience"; - my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}"; + my $workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}"; my $obs_mode = $obsmode_list{$target}; my $object = $object_list{$target}; @@ -837,4 +983,30 @@ } +sub get_dettool_parameters { + my $date = shift; + my $target = shift; + my $workdir_date = $date; $workdir_date =~ s%-%/%g; + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $exp_type = $exptype_list{$target}; + my $det_type = $dettype_list{$target}; + my $ref_det_id = $refID_list{$target}; + my $ref_iter = $refIter_list{$target}; + my $det_filter = $detfilter_list{$target}; + my $internal_filter; + if (defined($det_filter)) { + $internal_filter = $det_filter; $internal_filter =~ s/\..*//; + $internal_filter = '.' . $internal_filter; + } + else { + $internal_filter = ''; + } + my $maxN = $detmax_list{$target}; + + my $lc_type = lc($exp_type); + my $label = "${lc_type}${internal_filter}.$trunc_date"; + my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}"; + return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN); +} sub init_gpc_db { Index: trunk/ippTasks/nightly_stacks.pro =================================================================== --- trunk/ippTasks/nightly_stacks.pro (revision 28221) +++ trunk/ippTasks/nightly_stacks.pro (revision 28222) @@ -5,4 +5,5 @@ macro nightly.stacks.on ns.initday.on + ns.detrends.off ns.registration.on ns.burntool.on @@ -13,4 +14,5 @@ macro nightly.stacks.off ns.initday.off + ns.detrends.off ns.registration.off ns.burntool.off @@ -21,4 +23,5 @@ macro ns.on ns.initday.on + ns.detrends.off ns.registration.on ns.burntool.on @@ -29,4 +32,5 @@ macro ns.off ns.initday.off + ns.detrends.off ns.registration.off ns.burntool.off @@ -41,4 +45,10 @@ end +macro ns.detrends.on + task ns.detrends.load + active true + end +end + macro ns.registration.on task ns.registration.load @@ -76,4 +86,10 @@ macro ns.initday.off task ns.initday.load + active false + end +end + +macro ns.detrends.off + task ns.detrends.load active false end @@ -186,4 +202,39 @@ #operation times out? task.exit timeout + showcommand timeout + end +end + +# +# Queue off any possible detrend verification runs +# +task ns.detrends.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 23:00:00 24:00:00 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.detrends.log + stdout $LOGDIR/ns.detrends.log + $today = `date +%Y-%m-%d` + + command automate_stacks.pl --queue_detrends --date $today + end + + task.exit 0 + # nothign to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout showcommand timeout end Index: trunk/ippconfig/recipes/nightly_science.config =================================================================== --- trunk/ippconfig/recipes/nightly_science.config (revision 28221) +++ trunk/ippconfig/recipes/nightly_science.config (revision 28222) @@ -288,2 +288,75 @@ STACKABLE BOOL TRUE END + +# Detrend verification data +DETRENDS MULTI +DETRENDS METADATA + NAME STR DARK + DETTYPE STR DARK + EXPTYPE STR dark + REF_ID S64 299 + REF_ITER S32 0 +# FILTER STR + MAX S32 10 +END + +DETRENDS METADATA + NAME STR FLAT_G + DETTYPE STR FLAT + EXPTYPE STR skyflat + REF_ID S64 300 + REF_ITER S32 0 + FILTER STR g.00000 + MAX S32 10 +END + +DETRENDS METADATA + NAME STR FLAT_R + DETTYPE STR FLAT + EXPTYPE STR skyflat + REF_ID S64 301 + REF_ITER S32 0 + FILTER STR r.00000 + MAX S32 10 +END + +DETRENDS METADATA + NAME STR FLAT_I + DETTYPE STR FLAT + EXPTYPE STR skyflat + REF_ID S64 302 + REF_ITER S32 0 + FILTER STR i.00000 + MAX S32 10 +END + +DETRENDS METADATA + NAME STR FLAT_Z + DETTYPE STR FLAT + EXPTYPE STR skyflat + REF_ID S64 303 + REF_ITER S32 0 + FILTER STR z.00000 + MAX S32 10 +END + +DETRENDS METADATA + NAME STR FLAT_Y + DETTYPE STR FLAT + EXPTYPE STR skyflat + REF_ID S64 304 + REF_ITER S32 0 + FILTER STR y.00000 + MAX S32 10 +END + +DETRENDS METADATA + NAME STR FLAT_W + DETTYPE STR FLAT + EXPTYPE STR skyflat + REF_ID S64 305 + REF_ITER S32 0 + FILTER STR w.00000 + MAX S32 10 +END +