Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl (revision 28981) @@ -25,4 +25,5 @@ my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); +my $moondata = can_run('moondata') or (warn "Can't find moondata" and $missing_tools = 1); if ($missing_tools) { @@ -41,7 +42,7 @@ # Grab options my ( $date, $camera, $dbname, $logfile, $verbose, $manual); -my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode); +my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode, $burntool_stats); my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); -my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); +my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old); my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats); @@ -58,4 +59,5 @@ 'check' => \$check_mode, 'test_mode' => \$test_mode, + 'burntool_stats' => \$burntool_stats, 'this_target_only=s' => \$this_target_only, 'this_filter_only=s' => \$this_filter_only, @@ -68,4 +70,6 @@ 'check_stacks' => \$check_stacks, 'queue_stacks' => \$queue_stacks, + 'check_sweetspot' => \$check_sweetspot, + 'queue_sweetspot' => \$queue_sweetspot, 'check_detrends' => \$check_detrends, 'queue_detrends' => \$queue_detrends, @@ -89,4 +93,5 @@ --this_filter_only Process only a single filter. --this_mode_only Process only a single clean mode. + --burntool_stats Display Nexp Nimfile Nburntooled Nqueued for check_chips. Modes: --check_registration Confirm the data downloaded correctly. @@ -97,4 +102,6 @@ --check_stacks Confirm that stacks can be built. --queue_stacks Issue stacktool commands to queue stacks. + --check_sweetspot See if we should queue SweetSpot stacks. + --queue_sweetspot Issue stacktool commands to queue SweetSpot stacks. --check_detrends Confirm that detrend verify runs can be built. --queue_detrends Issue dettool commands to queue detrend verify runs. @@ -111,6 +118,6 @@ ) unless defined $check_registration or defined $define_burntool or defined $queue_burntool or - defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or - defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or + defined $queue_chips or defined $queue_stacks or $queue_sweetspot or $queue_detrends or $queue_dqstats or + defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or defined $test_mode or defined $clean_old or defined $check_mode; @@ -125,4 +132,5 @@ my %stackable_list = (); my %reduction_class = (); +my @unrecoverable_quality = (); my @detrend_list = (); my %dettype_list = (); @@ -170,4 +178,7 @@ elsif (${ $entry }{name} eq 'FILTERS') { push @filter_list, ${ $entry }{value}; + } + elsif (${ $entry }{name} eq 'UNRECOVERABLE_QUALITY') { + push @unrecoverable_quality, ${ $entry }{value}; } elsif (${ $entry }{name} eq 'TARGETS') { @@ -304,4 +315,5 @@ unless (defined($test_mode) || defined($check_mode)) { exit(0); } } + if (defined($check_dqstats) || defined($test_mode)) { $metadata_out{nsState} = 'CHECKDQSTATS'; @@ -316,4 +328,5 @@ unless (defined($test_mode)) { exit(0); } } + if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'CHECKDETRENDS'; @@ -328,4 +341,5 @@ exit(0); } + if (defined($define_burntool) || defined($test_mode)) { $metadata_out{nsState} = 'QUEUEBURNING'; @@ -339,4 +353,5 @@ unless (defined($test_mode)) { exit(0); } } + if (defined($check_chips) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'QUEUECHIPS'; @@ -354,4 +369,5 @@ exit(0); } + if (defined($check_stacks) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'TOWARP'; @@ -372,7 +388,22 @@ exit(0); } + +if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) { + $metadata_out{nsState} = 'CHECKSWEETSPOT'; + &execute_sweetspot($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_sweetspot)) { + $metadata_out{nsState} = 'CHECKSWEETSPOT'; + &execute_sweetspot($date); + return_metadata($date); + exit(0); +} + if (defined($check_diffs) || defined($queue_diffs)) { die("Diffs are currently not implemented."); } + if (defined($clean_old) || defined($test_mode)) { if (defined($test_mode)) { @@ -656,6 +687,11 @@ my $pretend = shift; my $exposures = 0; + foreach my $target (@target_list) { my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target); + if (defined($burntool_stats)) { + print "$Nexposures $Nimfiles $Nburntooled $Nalready\n"; + } + if ($Nexposures == 0) { print STDERR "execute_chips: Target $target on $date had no exposures.\n"; @@ -861,4 +897,134 @@ # +# SweetSpot Stacking +################################################################################ +sub construct_sweetspot_cmd { + my $date = shift; + my $target = 'SweetSpot'; + my $filter = 'w.00000'; + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + # Dateobs begin end? + my ($dateobs_begin,$dateobs_end) = get_lunation_extent($date); + my $select = "-select_dateobs_begin ${dateobs_begin}T00:00:00 -select_dateobs_end ${dateobs_end}T00:00:00"; + my $cmd = "$stacktool"; + $cmd .= " -simple -dbname $dbname -definebyquery "; + $cmd .= " -set_label SweetSpot.refstack -select_label $label "; + $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; + $cmd .= " -select_filter $filter -set_data_group $data_group "; + $cmd .= " -select_good_frac_min 0.1 -select_fwhm_major_max 8.0 "; + $cmd .= " -min_num 7 -min_new 4"; + $cmd .= " $select "; + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub get_lunation_extent { + my $date = shift; + my ($year,$month,$day) = split /-/,$date; + my $dateobs_begin; + my $dateobs_end; + + my $dt = DateTime->new(year => $year, month => $month, day => $day, + hour => 0, minute => 0, second => 0, nanosecond => 0, + time_zone => 'Pacific/Honolulu'); + do { + $dt->subtract(days => 1); + my $ymd = $dt->ymd; + my $md_cmd = "moondata $ymd 0 0"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $md_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + my @result = split /\s+/,(join "\n", @$stdout_buf); + if (abs($result[6]) <= 0.5) { + $dateobs_end = $ymd; + } + } while (!defined($dateobs_end)); + + do { + $dt->subtract(days => 1); + my $ymd = $dt->ymd; + my $md_cmd = "moondata $ymd 0 0"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $md_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + my @result = split /\s+/,(join "", @$stdout_buf); + if (abs($result[6]) <= 0.5) { + $dateobs_begin = $ymd; + } + } while (!defined($dateobs_begin)); + + return($dateobs_begin,$dateobs_end); +} + +sub pre_sweetspot_queue { + my $date = shift; + my $command = construct_sweetspot_cmd($date) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + my @stacks = split /\n/, (join '', @$stdout_buf); + my $Nstacks = $#stacks + 1; + + return(1,$Nstacks); +} + +sub sweetspot_queue { + my $date = shift; + my $command = construct_sweetspot_cmd($date); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + my @stacks = split /\n/, (join '', @$stdout_buf); + my $Nstacks = $#stacks + 1; + + return(1,$Nstacks); +} + +sub execute_sweetspot { + my $date = shift; + my $pretend = shift; + + my ($is_lunation_date,$Nstacks) = pre_sweetspot_queue($date); + if ($Nstacks == 0) { + print STDERR "execute_sweetspot: No new stacks to make ($Nstacks)\n"; + $metadata_out{nsState} = 'SS_EMPTY'; + return(); + } + if ($Nstacks < 10) { + print STDERR "execute_sweetspot: Too few new stacks to make ($Nstacks)\n"; + $metadata_out{nsState} = 'SS_FEW'; + return(); + } + if ($is_lunation_date == 0) { + print STDERR "execute_sweetspot: Invalid lunation date. ($date $is_lunation_date)\n"; + $metadata_out{nsState} = 'SS_ERROR'; + return(); + } + $metadata_out{nsState} = 'SS_QUEUE'; + $metadata_out{nsSweetSpot} = $Nstacks; + unless(defined($pretend)) { + $metadata_out{nsState} = 'SS_DONE'; + sweetspot_queue($date); + } +} + +# # Stacking ################################################################################ @@ -900,5 +1066,5 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); - my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'"; + my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter' AND tess_id = '$tess_id'"; my $data_ref = $db->selectall_arrayref( $sth ); @@ -918,8 +1084,17 @@ my $where = " label = '$label' AND data_group = '$data_group' "; + + my $where_possibly_faulted = $where . " AND ( " ; + foreach my $acceptable_quality (@unrecoverable_quality) { + $where_possibly_faulted .= " quality = $acceptable_quality OR "; + } + $where_possibly_faulted .= " 0 )"; + my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where "; + my $cam_sth = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted "; my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where "; my $chip_ref = $db->selectall_arrayref( $chip_sth ); + my $cam_ref = $db->selectall_arrayref( $cam_sth ); my $warp_ref = $db->selectall_arrayref( $warp_sth ); @@ -942,5 +1117,5 @@ } - return($#input_exposures + 1, $#{ $chip_ref } + 1, $#{ $warp_ref } + 1, $Nalready); + return($#input_exposures + 1, $#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2, $Nalready); } @@ -1039,5 +1214,5 @@ my $args = $command; if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) { - $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date "; + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -label $label -time_stamp_end $cleaning_date "; } elsif ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'B')) { @@ -1058,21 +1233,21 @@ foreach my $mode (@mode_list) { - if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) { - my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode); - if ($cleaning_date eq 'no clean') { - next; - } - print STDERR "$command\n"; - if (!(defined($pretend) || $debug == 1)) { -# print STDERR "BEAR IS DRIVING!?\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } - } - else { +# if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) { +# my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode); +# if ($cleaning_date eq 'no clean') { +# next; +# } +# print STDERR "$command\n"; +# if (!(defined($pretend) || $debug == 1)) { +# # print STDERR "BEAR IS DRIVING!?\n"; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# } +# } +# else { foreach my $target (@target_list) { if (exists($noclean_list{$target})) { @@ -1094,5 +1269,5 @@ } } - } +# } } return(0); Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/chip_imfile.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/chip_imfile.pl (revision 28981) @@ -23,4 +23,5 @@ use PS::IPP::Config 1.01 qw( :standard ); use File::Temp qw( tempfile ); +use File::Basename; use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); @@ -125,4 +126,11 @@ $outputStats .= '.update'; $traceDest .= '.update'; + # make sure that any lingering destreak backup files are gone + $ipprc->delete_destreak_backup_file($outputImage) + or &my_die("failed to delete existing destreak backup image file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR); + $ipprc->delete_destreak_backup_file($outputMask) + or &my_die("failed to delete existing destreak backup mask file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR); + $ipprc->delete_destreak_backup_file($outputWeight) + or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -454,4 +462,5 @@ } + sub my_die { Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_advancerun.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_advancerun.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_advancerun.pl (revision 28981) @@ -67,4 +67,5 @@ my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); if ($missing_tools) { &my_die("Can't find required tools.", $dist_id, $PS_EXIT_CONFIG_ERROR); @@ -108,4 +109,8 @@ $list_mode = "-sumskyfile"; $component_key = "skycell_id"; +} elsif ($stage eq "sky") { + $tool_cmd = "$staticskytool -sky_id"; + $list_mode = "-result"; + $component_key = ""; } elsif ($stage eq "diff") { $tool_cmd = "$difftool -diff_id"; Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_bundle.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_bundle.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_bundle.pl (revision 28981) @@ -53,4 +53,7 @@ 'PPSTACK.OUTPUT.MASK' => 'mask', 'PPSTACK.OUTPUT.VARIANCE' => 'variance' ); +my %sky_cleaned = ( 'PSPHOT.STACK.OUTPUT.IMAGE' => 'image', + 'PSPHOT.STACK.OUTPUT.MASK' => 'mask', + 'PSPHOT.STACK.OUTPUT.VARIANCE' => 'variance' ); @@ -58,4 +61,5 @@ my $missing_tools; my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); +my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -224,4 +228,48 @@ } +if (!$clean) { + if ($stage eq 'chip_bg') { + # add the variance file from the original chipRun + my $command = "$bgtool -listchip -chip_bg_id $stage_id -class_id $component"; + $command .= " -dbname $dbname" if $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $component, $error_code); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $list = $mdcParser->parse( join "", @$stdout_buf) or + &my_die("Unable to parse bgtool metadata", $component, $PS_EXIT_SYS_ERROR); + my $parsed = parse_md_list($list); + + # result is an array with one element + my $chip_data = $parsed->[0]; + + my $chip_path_base = $chip_data->{path_base}; + $variance = $ipprc->filename('PPIMAGE.CHIP.VARIANCE', $chip_path_base, $component) or + &my_die("Unable to resolve variance file name from $chip_path_base", $component, $PS_EXIT_SYS_ERROR); + + } elsif ($stage eq 'warp_bg') { + # add the variance file from the original warpRun + my $command = "$bgtool -listwarp -warp_bg_id $stage_id -skycell_id $component"; + $command .= " -dbname $dbname" if $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $component, $error_code); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $list = $mdcParser->parse( join "", @$stdout_buf) or + &my_die("Unable to parse bgtool metadata", $component, $PS_EXIT_SYS_ERROR); + my $parsed = parse_md_list($list); + my $warp_data = $parsed->[0]; + my $warp_path_base = $warp_data->{path_base}; + $variance = $ipprc->filename('PSWARP.OUTPUT.VARIANCE', $warp_path_base) or + &my_die("Unable to resolve variance file name from $warp_path_base", $component, $PS_EXIT_SYS_ERROR); + } +} + if ($nan_masked_pixels) { # One last check as to whether magic has been applied to the inputs @@ -239,4 +287,6 @@ if (($stage ne "chip_bg") and ($stage ne "warp_bg")) { &my_die("no variance image found in file list", $component, $PS_EXIT_CONFIG_ERROR) if !$variance; + } else { + &my_die("variance is not defined", $component, $PS_EXIT_CONFIG_ERROR) if !$variance; } } @@ -373,4 +423,6 @@ } elsif ($stage eq "stack") { $type = $stack_cleaned{$rule}; + } elsif ($stage eq "sky") { + $type = $sky_cleaned{$rule}; } else { &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR); @@ -453,4 +505,6 @@ } elsif ($stage eq "stack") { $config_file_rule = "PPSTACK.CONFIG"; + } elsif ($stage eq "sky") { + $config_file_rule = "PSPHOT.STACK.CONFIG"; } else { &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR); Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_make_fileset.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_make_fileset.pl (revision 28981) @@ -223,5 +223,5 @@ my $dbname = shift; - if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff')) { + if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky') { return ""; } Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/magic_destreak.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/magic_destreak.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/magic_destreak.pl (revision 28981) @@ -216,5 +216,5 @@ foreach my $skycell (@$skycells) { my $skycell_uri; - if ($skycell->{data_state} eq "full") { + if ($skycell->{data_state} eq "full" and $skycell->{quality} = 0) { $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base}); } else { Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/receive_advance.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/receive_advance.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/receive_advance.pl (revision 28981) @@ -69,4 +69,6 @@ } elsif ($stage eq "camera") { $tool_name = "camtool"; + } elsif ($stage eq "chip_bg" or $stage = "warp_bg") { + $tool_name = "bgtool"; } else { $tool_name = "${stage}tool"; Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/receive_file.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/receive_file.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/receive_file.pl (revision 28981) @@ -183,4 +183,7 @@ $stage = 'raw'; $comp_name = 'class_id'; + } elsif ($runType eq 'chipBackgroundRun') { + $stage = 'chip_bg'; + $comp_name = 'class_id'; } elsif ($runType eq 'chipRun') { $stage = 'chip'; @@ -195,4 +198,7 @@ } elsif ($runType eq 'warpRun') { $stage = 'warp'; + $comp_name = 'skycell_id'; + } elsif ($runType eq 'warpBackgroundRun') { + $stage = 'warp_bg'; $comp_name = 'skycell_id'; } elsif ($runType eq 'diffRun') { Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/staticsky.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippScripts/scripts/staticsky.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/staticsky.pl (revision 28981) @@ -99,4 +99,6 @@ my $nInputs = @$files; +my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); + foreach my $file (@$files) { print $listFile "INPUT METADATA\n"; @@ -161,5 +163,5 @@ $command .= " -threads $threads" if defined $threads; $command .= " -recipe PSPHOT $recipe_psphot"; - # $command .= " -dumpconfig $configuration"; + $command .= " -dumpconfig $configuration"; # $command .= " -tracedest $traceDest -log $logDest"; # $command .= " -dbname $dbname" if defined $dbname; Index: /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro =================================================================== --- /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro (revision 28981) @@ -7,4 +7,5 @@ ns.detrends.off ns.dqstats.on + ns.sweetspot.on ns.registration.on ns.burntool.on @@ -17,4 +18,5 @@ ns.detrends.off ns.dqstats.off + ns.sweetspot.off ns.registration.off ns.burntool.off @@ -49,4 +51,10 @@ end +macro ns.sweetspot.on + task ns.sweetspot.load + active true + end +end + macro ns.registration.on task ns.registration.load @@ -96,4 +104,10 @@ macro ns.dqstats.off task ns.dqstats.load + active false + end +end + +macro ns.sweetspot.off + task ns.sweetspot.load active false end @@ -263,4 +277,39 @@ command automate_stacks.pl --queue_dqstats --date $today + end + + task.exit 0 + # nothing to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Queue sweetspot runs +# +task ns.sweetspot.load + host local + periods -poll 3600 + periods -exec $LOADEXEC + periods -timeout 30 + trange 02:00:00 02:59:59 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.sweetspot.log + stderr $LOGDIR/ns.sweetspot.log + $today = `date +%Y-%m-%d` + + command automate_stacks.pl --queue_sweetspot --date $today end Index: /branches/eam_branches/ipp-20100621/ippTasks/survey.pro =================================================================== --- /branches/eam_branches/ipp-20100621/ippTasks/survey.pro (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTasks/survey.pro (revision 28981) @@ -30,4 +30,6 @@ book create SURVEY_DIST book create SURVEY_ADDSTAR + book create SURVEY_CHIP_BG + book create SURVEY_WARP_BG $haveSurveyBooks = TRUE end @@ -40,4 +42,6 @@ $SURVEY_DIST_DB = 0 $SURVEY_ADDSTAR_DB = 0 +$SURVEY_CHIP_BG_DB = 0 +$SURVEY_WARP_BG_DB = 0 $SURVEY_EXEC = 120 @@ -67,4 +71,10 @@ active true end + task survey.chip.bg + active true + end + task survey.warp.bg + active true + end end @@ -89,4 +99,10 @@ end task survey.dist + active false + end + task survey.chip.bg + active false + end + task survey.warp.bg active false end @@ -286,4 +302,57 @@ end +# user functions to manipulate chip_bg labels +macro survey.add.chip.bg + if ($0 != 3) + echo "USAGE: survey.add.chip.bg (label) (dist_group)" + break + end + book newpage SURVEY_CHIP_BG $1 + book setword SURVEY_CHIP_BG $1 DIST_GROUP $2 + book setword SURVEY_CHIP_BG $1 STATE PENDING +end + +macro survey.del.chip.bg + if ($0 != 2) + echo "USAGE: survey.del.chip.bg (label)" + break + end + book delpage SURVEY_CHIP_BG $1 +end + +macro survey.show.chip.bg + if ($0 != 1) + echo "USAGE: survey.show.chip.bg" + break + end + book listbook SURVEY_CHIP_BG +end + +# user functions to manipulate warp_bg labels +macro survey.add.warp.bg + if ($0 != 3) + echo "USAGE: survey.add.warp.bg (label) (dist_group)" + break + end + book newpage SURVEY_WARP_BG $1 + book setword SURVEY_WARP_BG $1 DIST_GROUP $2 + book setword SURVEY_WARP_BG $1 STATE PENDING +end + +macro survey.del.warp.bg + if ($0 != 2) + echo "USAGE: survey.del.warp.bg (label)" + break + end + book delpage SURVEY_WARP_BG $1 +end + +macro survey.show.warp.bg + if ($0 != 1) + echo "USAGE: survey.show.warp.bg" + break + end + book listbook SURVEY_WARP_BG +end task survey.diff @@ -815,2 +884,142 @@ end end + +task survey.chip.bg + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.chip.bg.log + stderr $LOGDIR/survey.chip.bg.log + + # generate chip_bg runs + task.exec + book npages SURVEY_CHIP_BG -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_CHIP_BG 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_CHIP_BG $i -var label + book setword SURVEY_CHIP_BG $label STATE NEW + end + book getpage SURVEY_CHIP_BG 0 -var label -key STATE NEW + + # Select different database + $SURVEY_CHIP_BG ++ + if ($SURVEY_CHIP_BG >= $DB:n) set SURVEY_CHIP_BG = 0 + end + + book setword SURVEY_CHIP_BG $label STATE DONE + book getword SURVEY_CHIP_BG $label DIST_GROUP -var dist_group + + $run = bgtool -definechip -label $label -set_dist_group $dist_group + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_CHIP_BG + option $DB:$SURVEY_CHIP_BG + end + + echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.warp.bg + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.warp.bg.log + stderr $LOGDIR/survey.warp.bg.log + + # generate warp_bg runs + task.exec + book npages SURVEY_WARP_BG -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_WARP_BG 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_WARP_BG $i -var label + book setword SURVEY_WARP_BG $label STATE NEW + end + book getpage SURVEY_WARP_BG 0 -var label -key STATE NEW + + # Select different database + $SURVEY_WARP_BG ++ + if ($SURVEY_WARP_BG >= $DB:n) set SURVEY_WARP_BG = 0 + end + + book setword SURVEY_WARP_BG $label STATE DONE + book getword SURVEY_WARP_BG $label DIST_GROUP -var dist_group + + $run = bgtool -definewarp -warp_label $label -set_dist_group $dist_group + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_WARP_BG + option $DB:$SURVEY_WARP_BG + end + + echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end