Changeset 28981

Timestamp:

Aug 19, 2010, 6:52:05 PM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/ipp-20100621

Files:

• ippScripts/scripts/automate_stacks.pl (modified) (23 diffs)
• ippScripts/scripts/chip_imfile.pl (modified) (3 diffs)
• ippScripts/scripts/dist_advancerun.pl (modified) (2 diffs)
• ippScripts/scripts/dist_bundle.pl (modified) (6 diffs)
• ippScripts/scripts/dist_make_fileset.pl (modified) (1 diff)
• ippScripts/scripts/magic_destreak.pl (modified) (1 diff)
• ippScripts/scripts/receive_advance.pl (modified) (1 diff)
• ippScripts/scripts/receive_file.pl (modified) (2 diffs)
• ippScripts/scripts/staticsky.pl (modified) (2 diffs)
• ippTasks (modified) (1 prop)
• ippTasks/nightly_stacks.pro (modified) (5 diffs)
• ippTasks/survey.pro (modified) (6 diffs)

branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl

@@ -25 +25 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1);
+my $moondata = can_run('moondata') or (warn "Can't find moondata" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -41 +42 @@
 # Grab options
 my ( $date, $camera, $dbname, $logfile, $verbose, $manual);
-my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
+my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode, $burntool_stats);
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
-my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
+my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old);
 my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats);
 
@@ -58 +59 @@
     'check'                => \$check_mode,
     'test_mode'            => \$test_mode,
+    'burntool_stats'       => \$burntool_stats,
     'this_target_only=s'   => \$this_target_only,
     'this_filter_only=s'   => \$this_filter_only,
@@ -68 +70 @@
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
+    'check_sweetspot'      => \$check_sweetspot,
+    'queue_sweetspot'      => \$queue_sweetspot,
     'check_detrends'       => \$check_detrends,
     'queue_detrends'       => \$queue_detrends,
@@ -89 +93 @@
            --this_filter_only     Process only a single filter.
            --this_mode_only       Process only a single clean mode.
+           --burntool_stats       Display Nexp Nimfile Nburntooled Nqueued for check_chips.
         Modes:
            --check_registration   Confirm the data downloaded correctly.
@@ -97 +102 @@
            --check_stacks         Confirm that stacks can be built.
            --queue_stacks         Issue stacktool commands to queue stacks.
+           --check_sweetspot      See if we should queue SweetSpot stacks.
+           --queue_sweetspot      Issue stacktool commands to queue SweetSpot stacks.
            --check_detrends       Confirm that detrend verify runs can be built.
            --queue_detrends       Issue dettool commands to queue detrend verify runs.
@@ -111 +118 @@
           ) unless
     defined $check_registration or defined $define_burntool or defined $queue_burntool or
-    defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or
-    defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or
+    defined $queue_chips or defined $queue_stacks or $queue_sweetspot or $queue_detrends or $queue_dqstats or 
+    defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or
     defined $test_mode or defined $clean_old or defined $check_mode;
 
@@ -125 +132 @@
 my %stackable_list = ();
 my %reduction_class = ();
+my @unrecoverable_quality = ();
 my @detrend_list = ();
 my %dettype_list = ();
@@ -170 +178 @@
     elsif (${ $entry }{name} eq 'FILTERS') {
         push @filter_list, ${ $entry }{value};
+    }
+    elsif (${ $entry }{name} eq 'UNRECOVERABLE_QUALITY') {
+        push @unrecoverable_quality, ${ $entry }{value};
     }
     elsif (${ $entry }{name} eq 'TARGETS') {
@@ -304 +315 @@
     unless (defined($test_mode) || defined($check_mode)) { exit(0); }
 }
+
 if (defined($check_dqstats) || defined($test_mode)) {
     $metadata_out{nsState} = 'CHECKDQSTATS';
@@ -316 +328 @@
     unless (defined($test_mode)) { exit(0); }
 }
+
 if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsState} = 'CHECKDETRENDS';
@@ -328 +341 @@
     exit(0);
 }
+
 if (defined($define_burntool) || defined($test_mode)) {
     $metadata_out{nsState} = 'QUEUEBURNING';
@@ -339 +353 @@
     unless (defined($test_mode)) { exit(0); }
 }
+
 if (defined($check_chips) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsState} = 'QUEUECHIPS';
@@ -354 +369 @@
     exit(0);
 }
+
 if (defined($check_stacks) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsState} = 'TOWARP';
@@ -372 +388 @@
     exit(0);
 }
+
+if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) {
+    $metadata_out{nsState} = 'CHECKSWEETSPOT';
+    &execute_sweetspot($date,"pretend");
+    return_metadata($date);
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
+}
+if (defined($queue_sweetspot)) {
+    $metadata_out{nsState} = 'CHECKSWEETSPOT';
+    &execute_sweetspot($date);
+    return_metadata($date);
+    exit(0);
+}
+
 if (defined($check_diffs) || defined($queue_diffs)) {
     die("Diffs are currently not implemented.");
 }
+
 if (defined($clean_old) || defined($test_mode)) {
     if (defined($test_mode)) {
@@ -656 +687 @@
     my $pretend = shift;
     my $exposures = 0;
+
     foreach my $target (@target_list) {
         my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target);
+        if (defined($burntool_stats)) {
+            print "$Nexposures $Nimfiles $Nburntooled $Nalready\n";
+        }
+
         if ($Nexposures == 0) {
             print STDERR "execute_chips: Target $target on $date had no exposures.\n";
@@ -861 +897 @@
 
 #
+# SweetSpot Stacking
+################################################################################
+sub construct_sweetspot_cmd {
+    my $date = shift;
+    my $target = 'SweetSpot';
+    my $filter = 'w.00000';
+
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    # Dateobs begin end?
+    my ($dateobs_begin,$dateobs_end) = get_lunation_extent($date);
+    my $select = "-select_dateobs_begin ${dateobs_begin}T00:00:00 -select_dateobs_end ${dateobs_end}T00:00:00";
+    my $cmd = "$stacktool";
+    $cmd .= " -simple -dbname $dbname -definebyquery ";
+    $cmd .= " -set_label SweetSpot.refstack -select_label $label ";
+    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
+    $cmd .= " -select_filter $filter -set_data_group $data_group ";
+    $cmd .= " -select_good_frac_min 0.1 -select_fwhm_major_max 8.0 ";
+    $cmd .= " -min_num 7 -min_new 4";
+    $cmd .= " $select ";
+    if ($debug == 1) {
+        $cmd .= ' -pretend ';
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}
+
+sub get_lunation_extent {
+    my $date = shift;
+    my ($year,$month,$day) = split /-/,$date;
+    my $dateobs_begin;
+    my $dateobs_end;
+
+    my $dt = DateTime->new(year => $year, month => $month, day => $day,
+                           hour => 0, minute => 0, second => 0, nanosecond => 0,
+                           time_zone => 'Pacific/Honolulu');
+    do {
+        $dt->subtract(days => 1);
+        my $ymd = $dt->ymd;
+        my $md_cmd = "moondata $ymd 0 0";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run ( command => $md_cmd, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR);
+        }
+        my @result = split /\s+/,(join "\n", @$stdout_buf);
+        if (abs($result[6]) <= 0.5) {
+            $dateobs_end = $ymd;
+        }
+    } while (!defined($dateobs_end));
+    
+    do {
+        $dt->subtract(days => 1);
+        my $ymd = $dt->ymd;
+        my $md_cmd = "moondata $ymd 0 0";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run ( command => $md_cmd, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR);
+        }
+        my @result = split /\s+/,(join "", @$stdout_buf);
+        if (abs($result[6]) <= 0.5) {
+            $dateobs_begin = $ymd;
+        }
+    } while (!defined($dateobs_begin));
+        
+    return($dateobs_begin,$dateobs_end);
+}
+
+sub pre_sweetspot_queue { 
+    my $date = shift;
+    my $command = construct_sweetspot_cmd($date) . ' -pretend ';
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose );
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    my @stacks = split /\n/, (join '', @$stdout_buf);
+    my $Nstacks = $#stacks + 1;
+        
+    return(1,$Nstacks);
+}
+
+sub sweetspot_queue {
+    my $date = shift;
+    my $command = construct_sweetspot_cmd($date);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose );
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    my @stacks = split /\n/, (join '', @$stdout_buf);
+    my $Nstacks = $#stacks + 1;
+    
+    return(1,$Nstacks);
+}
+
+sub execute_sweetspot {    
+    my $date = shift;
+    my $pretend = shift;
+    
+    my ($is_lunation_date,$Nstacks) = pre_sweetspot_queue($date);
+    if ($Nstacks == 0) {
+        print STDERR "execute_sweetspot: No new stacks to make ($Nstacks)\n";
+        $metadata_out{nsState} = 'SS_EMPTY';
+        return();
+    }
+    if ($Nstacks < 10) {
+        print STDERR "execute_sweetspot: Too few new stacks to make ($Nstacks)\n";
+        $metadata_out{nsState} = 'SS_FEW';
+        return();
+    }
+    if ($is_lunation_date == 0) {
+        print STDERR "execute_sweetspot: Invalid lunation date. ($date $is_lunation_date)\n";
+        $metadata_out{nsState} = 'SS_ERROR';
+        return();
+    }
+    $metadata_out{nsState} = 'SS_QUEUE';
+    $metadata_out{nsSweetSpot} = $Nstacks;
+    unless(defined($pretend)) {
+        $metadata_out{nsState} = 'SS_DONE';
+        sweetspot_queue($date);
+    }
+}
+
+#
 # Stacking
 ################################################################################
@@ -900 +1066 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
 
-    my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'";
+    my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter' AND tess_id = '$tess_id'";
     my $data_ref = $db->selectall_arrayref( $sth );
 
@@ -918 +1084 @@
 
     my $where = " label = '$label' AND data_group = '$data_group' ";
+
+    my $where_possibly_faulted = $where . " AND ( " ;
+    foreach my $acceptable_quality (@unrecoverable_quality) {
+        $where_possibly_faulted .= " quality = $acceptable_quality OR ";
+    }
+    $where_possibly_faulted .= " 0 )";
+
     my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where ";
+    my $cam_sth  = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted ";
     my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where ";
 
     my $chip_ref = $db->selectall_arrayref( $chip_sth );
+    my $cam_ref  = $db->selectall_arrayref( $cam_sth );
     my $warp_ref = $db->selectall_arrayref( $warp_sth );
 
@@ -942 +1117 @@
     }
 
-    return($#input_exposures  + 1, $#{ $chip_ref } + 1, $#{ $warp_ref } + 1, $Nalready);
+    return($#input_exposures  + 1, $#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2, $Nalready);
 }
 
@@ -1039 +1214 @@
     my $args = $command;
     if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) {
-        $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date ";
+        $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -label $label -time_stamp_end $cleaning_date ";
     }
     elsif ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'B')) {
@@ -1058 +1233 @@
 
     foreach my $mode (@mode_list) {
-        if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) {
-            my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);
-            if ($cleaning_date eq 'no clean') {
-                next;
-            }
-            print STDERR "$command\n";
-            if (!(defined($pretend) || $debug == 1)) {
-#           print STDERR "BEAR IS DRIVING!?\n";
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run ( command => $command, verbose => $verbose );
-                unless ($success) {
-                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                    &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-                }
-            }
-        }
-        else {
+#       if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) {
+#           my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);
+#           if ($cleaning_date eq 'no clean') {
+#               next;
+#           }
+#           print STDERR "$command\n";
+#           if (!(defined($pretend) || $debug == 1)) {
+# #           print STDERR "BEAR IS DRIVING!?\n";
+#               my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#                   run ( command => $command, verbose => $verbose );
+#               unless ($success) {
+#                   $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#                   &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+#               }
+#           }
+#       }
+#       else {
             foreach my $target (@target_list) {
                 if (exists($noclean_list{$target})) {
@@ -1094 +1269 @@
                 }
             }
-        }
+#       }
     }
     return(0);

branches/eam_branches/ipp-20100621/ippScripts/scripts/chip_imfile.pl

@@ -23 +23 @@
 use PS::IPP::Config 1.01 qw( :standard );
 use File::Temp qw( tempfile );
+use File::Basename;
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -125 +126 @@
     $outputStats .= '.update';
     $traceDest .= '.update';
+    # make sure that any lingering destreak backup files are gone
+    $ipprc->delete_destreak_backup_file($outputImage)
+        or &my_die("failed to delete existing destreak backup image file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
+    $ipprc->delete_destreak_backup_file($outputMask)
+        or &my_die("failed to delete existing destreak backup mask file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
+    $ipprc->delete_destreak_backup_file($outputWeight)
+        or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
 }
 
@@ -454 +462 @@
 }
 
+
 sub my_die
 {

branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_advancerun.pl

@@ -67 +67 @@
 my $stacktool   = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
 if ($missing_tools) {
     &my_die("Can't find required tools.", $dist_id, $PS_EXIT_CONFIG_ERROR);
@@ -108 +109 @@
     $list_mode = "-sumskyfile";
     $component_key = "skycell_id";
+} elsif ($stage eq "sky") {
+    $tool_cmd = "$staticskytool -sky_id";
+    $list_mode = "-result";
+    $component_key = "";
 } elsif ($stage eq "diff") {
     $tool_cmd = "$difftool -diff_id";

branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_bundle.pl

@@ -53 +53 @@
                       'PPSTACK.OUTPUT.MASK' => 'mask',
                       'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
+my %sky_cleaned   = ( 'PSPHOT.STACK.OUTPUT.IMAGE' => 'image',
+                      'PSPHOT.STACK.OUTPUT.MASK' => 'mask',
+                      'PSPHOT.STACK.OUTPUT.VARIANCE' => 'variance' );
 
 
@@ -58 +61 @@
 my $missing_tools;
 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
+my $bgtool   = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -224 +228 @@
 }
 
+if (!$clean) {
+    if ($stage eq 'chip_bg') {
+        # add the variance file from the original chipRun
+        my $command = "$bgtool -listchip -chip_bg_id $stage_id -class_id $component";
+        $command .= " -dbname $dbname" if $dbname;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $component, $error_code);
+        }
+        my $mdcParser = PS::IPP::Metadata::Config->new; 
+        my $list = $mdcParser->parse( join "", @$stdout_buf) or 
+            &my_die("Unable to parse bgtool metadata", $component, $PS_EXIT_SYS_ERROR);
+        my $parsed = parse_md_list($list);
+
+        # result is an array with one element
+        my $chip_data = $parsed->[0];
+
+        my $chip_path_base = $chip_data->{path_base};
+        $variance = $ipprc->filename('PPIMAGE.CHIP.VARIANCE', $chip_path_base, $component) or
+            &my_die("Unable to resolve variance file name from $chip_path_base", $component, $PS_EXIT_SYS_ERROR);
+
+    } elsif ($stage eq 'warp_bg') {
+        # add the variance file from the original warpRun
+        my $command = "$bgtool -listwarp -warp_bg_id $stage_id -skycell_id $component";
+        $command .= " -dbname $dbname" if $dbname;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $component, $error_code);
+        }
+        my $mdcParser = PS::IPP::Metadata::Config->new; 
+        my $list = $mdcParser->parse( join "", @$stdout_buf) or 
+            &my_die("Unable to parse bgtool metadata", $component, $PS_EXIT_SYS_ERROR);
+        my $parsed = parse_md_list($list);
+        my $warp_data = $parsed->[0];
+        my $warp_path_base = $warp_data->{path_base};
+        $variance = $ipprc->filename('PSWARP.OUTPUT.VARIANCE', $warp_path_base) or
+            &my_die("Unable to resolve variance file name from $warp_path_base", $component, $PS_EXIT_SYS_ERROR);
+    }
+}
+
 if ($nan_masked_pixels) {
     # One last check as to whether magic has been applied to the inputs
@@ -239 +287 @@
         if (($stage ne "chip_bg") and ($stage ne "warp_bg")) {
             &my_die("no variance image found in file list", $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
+        } else {
+            &my_die("variance is not defined", $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
         }
     }
@@ -373 +423 @@
     } elsif ($stage eq "stack") {
         $type = $stack_cleaned{$rule};
+    } elsif ($stage eq "sky") {
+        $type = $sky_cleaned{$rule};
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -453 +505 @@
     } elsif ($stage eq "stack") {
         $config_file_rule = "PPSTACK.CONFIG";
+    } elsif ($stage eq "sky") {
+        $config_file_rule = "PSPHOT.STACK.CONFIG";
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);

branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_make_fileset.pl

@@ -223 +223 @@
     my $dbname = shift;
 
-    if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff')) {
+    if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky') {
         return "";
     }

branches/eam_branches/ipp-20100621/ippScripts/scripts/magic_destreak.pl

@@ -216 +216 @@
             foreach my $skycell (@$skycells) {
                 my $skycell_uri;
-                if ($skycell->{data_state} eq "full") {
+                if ($skycell->{data_state} eq "full" and $skycell->{quality} = 0) {
                     $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
                 } else {

branches/eam_branches/ipp-20100621/ippScripts/scripts/receive_advance.pl

@@ -69 +69 @@
     } elsif ($stage eq "camera") {
         $tool_name = "camtool";
+    } elsif ($stage eq "chip_bg" or $stage = "warp_bg") {
+        $tool_name = "bgtool";
     } else {
         $tool_name = "${stage}tool";

branches/eam_branches/ipp-20100621/ippScripts/scripts/receive_file.pl

@@ -183 +183 @@
         $stage = 'raw';
         $comp_name = 'class_id';
+    } elsif ($runType eq 'chipBackgroundRun') {
+        $stage = 'chip_bg';
+        $comp_name = 'class_id';
     } elsif ($runType eq 'chipRun') {
         $stage = 'chip';
@@ -195 +198 @@
     } elsif ($runType eq 'warpRun') {
         $stage = 'warp';
+        $comp_name = 'skycell_id';
+    } elsif ($runType eq 'warpBackgroundRun') {
+        $stage = 'warp_bg';
         $comp_name = 'skycell_id';
     } elsif ($runType eq 'diffRun') {

branches/eam_branches/ipp-20100621/ippScripts/scripts/staticsky.pl

@@ -99 +99 @@
 my $nInputs = @$files;
 
+my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot);
+
 foreach my $file (@$files) {
     print $listFile "INPUT   METADATA\n";
@@ -161 +163 @@
     $command .= " -threads $threads" if defined $threads;
     $command .= " -recipe PSPHOT $recipe_psphot";
-    # $command .= " -dumpconfig $configuration";
+    $command .= " -dumpconfig $configuration";
     # $command .= " -tracedest $traceDest -log $logDest";
     # $command .= " -dbname $dbname" if defined $dbname;

branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro

@@ -7 +7 @@
     ns.detrends.off
     ns.dqstats.on
+    ns.sweetspot.on
     ns.registration.on
     ns.burntool.on
@@ -17 +18 @@
     ns.detrends.off
     ns.dqstats.off
+    ns.sweetspot.off
     ns.registration.off
     ns.burntool.off
@@ -49 +51 @@
 end
 
+macro ns.sweetspot.on
+  task ns.sweetspot.load
+    active true
+  end
+end
+
 macro ns.registration.on
   task ns.registration.load
@@ -96 +104 @@
 macro ns.dqstats.off
   task ns.dqstats.load
+    active false
+  end
+end
+
+macro ns.sweetspot.off
+  task ns.sweetspot.load
     active false
   end
@@ -263 +277 @@
 
     command automate_stacks.pl --queue_dqstats --date $today
+  end
+
+  task.exit       0
+    # nothing to do here
+  end
+  # locked list
+  task.exit       default
+    showcommand failure
+  end
+  task.exit       crash
+    showcommand crash
+  end
+  # operation times out
+  task.exit       timeout
+    showcommand timeout
+  end
+end
+
+#
+# Queue sweetspot runs
+#
+task              ns.sweetspot.load
+  host            local
+  periods         -poll 3600
+  periods         -exec $LOADEXEC
+  periods         -timeout 30
+  trange          02:00:00 02:59:59 -nmax 1
+  npending        1
+
+  task.exec
+    stdout $LOGDIR/ns.sweetspot.log
+    stderr $LOGDIR/ns.sweetspot.log
+    $today = `date +%Y-%m-%d`
+
+    command automate_stacks.pl --queue_sweetspot --date $today
   end
 

branches/eam_branches/ipp-20100621/ippTasks/survey.pro

@@ -30 +30 @@
  book create SURVEY_DIST
  book create SURVEY_ADDSTAR   
+ book create SURVEY_CHIP_BG
+ book create SURVEY_WARP_BG
  $haveSurveyBooks = TRUE
 end
@@ -40 +42 @@
 $SURVEY_DIST_DB = 0
 $SURVEY_ADDSTAR_DB = 0
+$SURVEY_CHIP_BG_DB = 0
+$SURVEY_WARP_BG_DB = 0
 
 $SURVEY_EXEC = 120
@@ -67 +71 @@
     active true
   end
+  task survey.chip.bg
+    active true
+  end
+  task survey.warp.bg
+    active true
+  end
 end
 
@@ -89 +99 @@
   end
   task survey.dist
+    active false
+  end
+  task survey.chip.bg
+    active false
+  end
+  task survey.warp.bg
     active false
   end
@@ -286 +302 @@
 end
 
+# user functions to manipulate chip_bg labels
+macro survey.add.chip.bg
+  if ($0 != 3)
+    echo "USAGE: survey.add.chip.bg (label) (dist_group)"
+    break
+  end
+  book newpage SURVEY_CHIP_BG $1
+  book setword SURVEY_CHIP_BG $1 DIST_GROUP $2
+  book setword SURVEY_CHIP_BG $1 STATE PENDING
+end
+
+macro survey.del.chip.bg
+  if ($0 != 2)
+    echo "USAGE: survey.del.chip.bg (label)"
+    break
+  end
+  book delpage SURVEY_CHIP_BG $1
+end
+
+macro survey.show.chip.bg
+  if ($0 != 1)
+    echo "USAGE: survey.show.chip.bg"
+    break
+  end
+  book listbook SURVEY_CHIP_BG
+end
+
+# user functions to manipulate warp_bg labels
+macro survey.add.warp.bg
+  if ($0 != 3)
+    echo "USAGE: survey.add.warp.bg (label) (dist_group)"
+    break
+  end
+  book newpage SURVEY_WARP_BG $1
+  book setword SURVEY_WARP_BG $1 DIST_GROUP $2
+  book setword SURVEY_WARP_BG $1 STATE PENDING
+end
+
+macro survey.del.warp.bg
+  if ($0 != 2)
+    echo "USAGE: survey.del.warp.bg (label)"
+    break
+  end
+  book delpage SURVEY_WARP_BG $1
+end
+
+macro survey.show.warp.bg
+  if ($0 != 1)
+    echo "USAGE: survey.show.warp.bg"
+    break
+  end
+  book listbook SURVEY_WARP_BG
+end
 
 task survey.diff
@@ -815 +884 @@
   end
 end
+
+task survey.chip.bg
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.chip.bg.log
+  stderr $LOGDIR/survey.chip.bg.log
+
+  # generate chip_bg runs
+  task.exec
+    book npages SURVEY_CHIP_BG -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_CHIP_BG 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_CHIP_BG $i -var label
+        book setword SURVEY_CHIP_BG $label STATE NEW
+      end
+      book getpage SURVEY_CHIP_BG 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_CHIP_BG ++
+      if ($SURVEY_CHIP_BG >= $DB:n) set SURVEY_CHIP_BG = 0
+    end
+
+    book setword SURVEY_CHIP_BG $label STATE DONE
+    book getword SURVEY_CHIP_BG $label DIST_GROUP -var dist_group
+  
+    $run = bgtool -definechip -label $label -set_dist_group $dist_group
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_CHIP_BG
+      option $DB:$SURVEY_CHIP_BG
+    end
+    
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task survey.warp.bg
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.warp.bg.log
+  stderr $LOGDIR/survey.warp.bg.log
+
+  # generate warp_bg runs
+  task.exec
+    book npages SURVEY_WARP_BG -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_WARP_BG 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_WARP_BG $i -var label
+        book setword SURVEY_WARP_BG $label STATE NEW
+      end
+      book getpage SURVEY_WARP_BG 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_WARP_BG ++
+      if ($SURVEY_WARP_BG >= $DB:n) set SURVEY_WARP_BG = 0
+    end
+
+    book setword SURVEY_WARP_BG $label STATE DONE
+    book getword SURVEY_WARP_BG $label DIST_GROUP -var dist_group
+  
+    $run = bgtool -definewarp -warp_label $label -set_dist_group $dist_group
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_WARP_BG
+      option $DB:$SURVEY_WARP_BG
+    end
+    
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end