Changeset 30587 for branches/czw_branch/20101203/ippScripts

Timestamp:

Feb 11, 2011, 4:42:29 PM (15 years ago)

Author:

watersc1

Message:

Caught up with the trunk. Now to look for bugs.

Location:

branches/czw_branch/20101203

Files:

• . (modified) (1 prop)
• ippScripts/scripts/chip_imfile.pl (modified) (1 diff)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (5 diffs)
• ippScripts/scripts/ipp_cleanup.pl (modified) (6 diffs)
• ippScripts/scripts/magic_destreak_cleanup.pl (modified) (4 diffs)
• ippScripts/scripts/nightly_science.pl (modified) (3 diffs)
• ippScripts/scripts/register_exp.pl (modified) (3 diffs)
• ippScripts/scripts/warp_skycell.pl (modified) (9 diffs)

branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl

@@ -325 +325 @@
         # get the UNIX version of the (possible) neb: or path: filename
         my $uriReal = $ipprc->file_resolve( $uri );
+        &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
 
         # funpack into the temp file.

branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl

@@ -15 +15 @@
 my $REALRUN = 1;
 
-my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps, $rerun );
+my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps );
+my ( $rerun, $fixit );
+my $continue = 0;
 GetOptions(
     'exp_id=s'         => \$exp_id,   # exposure identifier
@@ -26 +28 @@
     'logfile=s'         => \$logfile,
     'rerun'             => \$rerun,
+#    'fix|f'             => \$fixit,
+    'continue=i'        => \$continue,
     'verbose'           => \$verbose,   # Print to stdout
     'save-temps'        => \$save_temps, # Save temporary files?
@@ -33 +37 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-    -msg => "Required options: --class_id --this_uri --dbname",
+    -msg => "Required options: --class_id --this_uri --exp_id --dbname",
     -exitval => 3,
     ) unless
     defined $class_id and
     defined $this_uri and
+    defined $exp_id and
     defined $dbname;
 
@@ -93 +98 @@
 print ">>$burntoolStateGood<<\n";
 
+# Determine what to do if we're fixing.
+# if ($fixit) {
+#     ($exp_id,$this_uri,$previous_uri,$continue) = fix_burntool($exp_id,$this_uri,$class_id);
+# }
+
+
+# Precalculate what we should be able to do if we're continuing
+my @continue_uris;
+my @continue_exp_ids;
+my @continue_states;
+if ($continue > 0) {
+    my $regtool_get_date   = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname";
+    my $dateobs;
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $regtool_get_date, verbose => $verbose );
+    unless ($success) {
+        $continue = 0;
+        last;
+    }
+    my $imfile_data = $mdcParser->parse(join "", @$stdout_buf);
+    my $imfile_data2 = parse_md_list($imfile_data);
+    if ($#{ $imfile_data2 } > 0) {
+        &my_die("Too many entries returned for date query", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+    foreach my $entry (@$imfile_data2) {
+        $dateobs = $entry->{dateobs};
+    }
+    my $date = $dateobs; $date =~ s/T.*//;
+
+    my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs -dateobs_end ${date}T17:30:00 -class_id $class_id -dbname $dbname";
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $regtool_status_cmd, verbose => $verbose );
+    unless ($success) {
+        $continue = 0;
+    }
+    if ($continue != 0) {
+        my $night_data = $mdcParser->parse(join "", @$stdout_buf);
+        my $night_data2 = parse_md_list($night_data);
+        my $addable = 0;
+        
+        foreach my $entry (@$night_data2) {
+            my ($this_exp_id,$this_uri,$this_state);
+            if ($entry->{exp_id} == $exp_id) {
+                $addable = 1;
+                next;
+            }
+            if (($addable == 1)&&($#continue_exp_ids < $continue)) {
+                $this_exp_id = $entry->{exp_id};
+                $this_uri    = $entry->{uri};
+                $this_state  = $entry->{imfile_state};
+                print ">> $this_exp_id $this_uri $this_state\n";
+                if ($this_state eq 'pending_burntool') {
+                    push @continue_exp_ids, $this_exp_id;
+                    push @continue_uris, $this_uri;
+                    push @continue_states, $this_state;
+                }
+                else {
+                    last;
+                }
+            }
+            if ($#continue_exp_ids >= $continue) {
+                last;
+            }
+        }
+    }
+}
+$continue = $#continue_exp_ids + 1;
+
+# Look over at least one uri/imfile/exp_id and do burntool on it.
+my $next_exp_id;
+for (my $iteration = 0; $iteration <= $continue; $iteration ++) {
 # Set up files
-my $rawImfile = $this_uri;
-my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
-
-my $outTable  = $this_uri;
-$outTable  =~ s/fits$/burn.tbl/;
-my $outTableReal = $ipprc->file_resolve($outTable, 1);
-
-my $previousTable;
-my $previousTableReal;
-if ($previous_uri) {
-    $previousTable = $previous_uri;
-    $previousTable =~ s/fits$/burn.tbl/;
-    $previousTableReal = $ipprc->file_resolve($previousTable, 0);
-}
-
+    my $rawImfile = $this_uri;
+    my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
+    
+    my $outTable  = $this_uri;
+    $outTable  =~ s/fits$/burn.tbl/;
+    my $outTableReal = $ipprc->file_resolve($outTable, 1);
+    
+    my $previousTable;
+    my $previousTableReal;
+    if ($previous_uri) {
+        $previousTable = $previous_uri;
+        $previousTable =~ s/fits$/burn.tbl/;
+        $previousTableReal = $ipprc->file_resolve($previousTable, 0);
+    }
+    
 # Set state to processing:
-my $burntool_state = 0;
-
-my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN);
-if ($status) {
-    &my_die("failed to update imfile");
-}
-$burntool_state = 1;
-
-
+    my $burntool_state = 0;
+    
+    my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN);
+    if ($status) {
+        &my_die("failed to update imfile");
+    }
+    $burntool_state = 1;
+    
+    
 # funpack
-my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX",
-                                DIR => '/tmp/',
-                                UNLINK => !$save_temps,
-                                SUFFIX => '.fits');
-my $tempPixels = $tempfile->filename;
-
-$status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
-if ($status) {
-    &my_die("failed on funpack",$exp_id,$class_id);
-}
-
+    my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX",
+                                    DIR => '/tmp/',
+                                    UNLINK => !$save_temps,
+                                    SUFFIX => '.fits');
+    my $tempPixels = $tempfile->filename;
+    
+    $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
+    if ($status) {
+        &my_die("failed on funpack",$exp_id,$class_id);
+    }
+    
 # burntool
-if (($previousTableReal)&&($previousTableReal ne '')) {
-    $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN);
-}
-else {
-    $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN);
-}
-if ($status) {
-    &my_die("failed on burntool",$exp_id,$class_id);
-}
-&my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal);
-
+    if (($previousTableReal)&&($previousTableReal ne '')) {
+        $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN);
+    }
+    else {
+        $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN);
+    }
+    if ($status) {
+        &my_die("failed on burntool",$exp_id,$class_id);
+    }
+    &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal);
+    
 # Replicate files
-$status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
-$status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
-
+    $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
+    $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
+
+# Prep for next iteration if necessary.
+    if ($iteration < $continue) {
+        $next_exp_id = shift(@continue_exp_ids);
+        my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $next_exp_id -class_id $class_id -set_state check_burntool";
+        $status = vsystem ($command, $REALRUN);
+        if ($status) {
+            $continue = 0;
+        }
+    }
 # Set state to finished.
-my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState";
-if ($imfile_state ne 'full') {
-    $command .= " -set_state full ";
-}
-$status = vsystem ($command, $REALRUN);
-if ($status) {
-    &my_die("failed to update imfile");
-}
-
+    my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState";
+    if ($imfile_state ne 'full') {
+        $command .= " -set_state full ";
+    }
+    $status = vsystem ($command, $REALRUN);
+    if ($status) {
+        &my_die("failed to update imfile");
+    }
+
+# Set values for the next iteration if we are doing another iteration.
+    $previous_uri = $this_uri;
+    if ($iteration < $continue) {
+        $this_uri = shift (@continue_uris);
+        $exp_id   = $next_exp_id;
+        $imfile_state = shift(@continue_states);
+    }
+
+}
 exit 0;
 
@@ -183 +277 @@
     exit 1;
 }
-
-
-    
+        
+# sub fix_burntool {
+#     my ($exp_id,$this_uri,$class_id,$burntoolGoodState) = @_;
+#     my ($dateobs,$exp_name);
+#     # Calculate the date for this exposure, and calculate back to find the burntool period.
+#     my $regtool_get_date   = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname";
+#     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#       run ( command => $regtool_get_date, verbose => $verbose );
+#     unless ($success) {
+#       $continue = 0;
+#       return($exp_id,$this_uri,undef,0);
+#     }
+#     my $imfile_data = $mdcParser->parse(join "", @$stdout_buf);
+#     foreach my $entry (@$imfile_data) {
+#       if ($entry->{name} eq 'dateobs') {
+#           $dateobs = $entry->{value};
+#       }
+#       if ($entry->{name} eq 'exp_name') {
+#           $exp_name = $entry->{value};
+#       }
+#     }
+#     my $date = $dateobs; $date =~ s/T.*//;
+#     my $time = $dateobs; $time =~ s/.*T//;
+#     my ($hour,$min,$sec) = split /\:/, $time; #/;
+#     if ($min >= 30) {
+#       $min -= 30;
+#     }
+#     else {
+#       $min += 30;
+#       $hour -= 1;
+#     }
+#     my $dateobs_begin = sprintf("%sT%02d:%02d:%02d",$date,$hour,$min,$sec);
+
+
+#     # Get the night data for this date range.
+#     my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs_begin -dateobs_end $dateobs -class_id $class_id -dbname $dbname";
+#     ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#       run ( command => $regtool_get_date, verbose => $verbose );
+#     unless ($success) {
+#       return($exp_id,$this_uri,undef,0);
+#     }
+
+#     my $night_data = $mdcParser->parse(join "", @$stdout_buf);
+
+#     my $mode;
+#     my ($row_exp_id,$row_uri,$row_data_state,$row_bt_state);
+#     my $entry = @$night_data[-1];
+#     foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
+#       if ($row->{name} eq 'exp_id') {
+#           $row_exp_id = $row->{value};
+#       }
+#       elsif ($row->{name} eq 'uri') {
+#           $row_uri = $row->{uri};
+#       }
+#       elsif ($row->{name} eq 'data_state') {
+#           $row_data_state = $row->{data_state};
+#       }               
+#       elsif ($row->{name} eq 'burntool_state') {
+#           $row_bt_state = $row->{burntool_state};
+#       }               
+#     }
+#     # Determine what to do.
+#     if ($row_exp_id == $exp_id) {
+#       if (($row_bt_state != $burntoolStateGood)) {
+#           # Never been run.
+#           $mode = 1;
+#       }
+#       else {
+#           $mode = 0;
+#       }
+#     }
+#     if ($mode == 0) {
+#       # If we're here, then the database thinks we have a table for this uri on disk. Let's see if that's true.
+#       # Zero byte file/good alternate
+#       # Inconsistent md5sums
+#       # Missing this table only.
+#       my $neb = $ipprc->nebulous();
+#       my $table_key = $this_uri; $table_key =~ s/fits/burn.tbl/;
+
+#       my $neb_instances = $neb->find_instances($table_key, 'any');
+
+#       my $Ninstances = $#{ $neb_instances } + 1;
+
+#       if (($Ninstances == 0)||($Ninstances == 1)) {
+#           #Missing this table only.  This should fall through to do a regular pass on this uri
+#           $continue = 0;
+#           $entry = @$night_data[-2];
+#           foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
+#               if ($row->{name} eq 'exp_id') {
+#                   $row_exp_id = $row->{value};
+#               }
+#               elsif ($row->{name} eq 'uri') {
+#                   $row_uri = $row->{uri};
+#               }
+#               elsif ($row->{name} eq 'data_state') {
+#                   $row_data_state = $row->{data_state};
+#               }               
+#               elsif ($row->{name} eq 'burntool_state') {
+#                   $row_bt_state = $row->{burntool_state};
+#               }               
+#           }
+#           $previous_uri = $row_uri;
+#           # Go through to regular processing of this exposure.
+#       }
+#       elsif ($Ninstances == 2) {
+#           $md5sum_A = md5sum(${ $neb_instances }[0]);
+#           $md5sum_B = md5sum(${ $neb_instances }[1]);
+#           if ($md5sum_A != $md5sum_B) {
+#               # md5sums are different
+#               if ($md5sum_A = 'd41d8cd98f00b204e9800998ecf8427e') {
+#                   # copy B -> A
+#               }
+#               elsif ($md5sum_B = 'd41d8cd98f00b204e9800998ecf8427e') {
+#                   # copy A -> B
+#               }
+#               else {
+#                   # different but unknown why
+#               }
+#           }
+#       }
+#     }
+
+#     if ($mode == 1) {
+#       foreach my $entry (reverse(@$night_data)) {  # reverse it so we can start with the exposure we have and go backwards
+#           foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
+#               if ($row->{name} eq 'exp_id') {
+#                   $row_exp_id = $row->{value};
+#               }
+#               elsif ($row->{name} eq 'uri') {
+#                   $row_uri = $row->{uri};
+#               }
+#               elsif ($row->{name} eq 'data_state') {
+#                   $row_data_state = $row->{data_state};
+#               }               
+#               elsif ($row->{name} eq 'burntool_state') {
+#                   $row_bt_state = $row->{burntool_state};
+#               }               
+#           }
+#       }
+#     }
+
+#     return($exp_id,$this_uri,$previous_uri,$continue);
+# }

branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl

@@ -207 +207 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -324 +315 @@
         }
 
-        # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-        my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-        $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-        $command .= " -dbname $dbname" if defined $dbname;
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-        }
+        set_destreak_goto_cleaned();
 
     } else {
@@ -469 +451 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
          } else {
@@ -603 +576 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -759 +723 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -2062 +2017 @@
 }
 
+# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
+my $ds_done = 0;
+sub set_destreak_goto_cleaned {
+
+    return if $ds_done;
+
+    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
+    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
+    my $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    $ds_done = 1;
+}
+
 # XXX we currently do not set the error state in the db on my_die
 sub my_die

branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl

@@ -159 +159 @@
         my $component = $comp->{component};
         my $backup_path_base = $comp->{backup_path_base};
-        my $recovery_path_base = $comp->{recovery_path_base};
         my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
         my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
@@ -197 +196 @@
                 $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $component);
             }
-            if ($recovery_path_base) {
-                $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component);
-
-                if ($dynamicMasks) {
-                    my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component);
-                    # This is kludgey but correct
-                    $rmask = dirname($recovery_path_base) . "/" . basename($mask);
-                    $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
-                } else {
-                    $rmask = $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
-                }
-                $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component);
-            }
         } elsif ($stage eq "camera") {
             if ($backup_path_base) {
@@ -221 +207 @@
                 $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
             }
-            if ($recovery_path_base) {
-                $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
-                $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
-                $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
-                $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base);
-            }
         } elsif ($stage eq "diff") {
             my $name = "PPSUB.OUTPUT";
+            if ($backup_path_base) {
+                $bimage  = $ipprc->filename($name, $backup_path_base);
+                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
+                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
+                # bills 2011-01-24
+                # don't clean up the uncensored Diff sources file
+                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+            }
+        }
+
+        delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
+
+        if ($stage eq "diff" and $warp_warp) {
+            my $name = "PPSUB.INVERSE";
             if ($backup_path_base) {
                 $bimage  = $ipprc->filename($name, $backup_path_base);
@@ -237 +231 @@
                 # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
             }
-            if ($recovery_path_base) {
-                $rimage  = $ipprc->filename($name, $recovery_path_base);
-                $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
-                $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
-                $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
-            }
-        }
-
-        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
-
-        if ($stage eq "diff" and $warp_warp) {
-            my $name = "PPSUB.INVERSE";
-            if ($backup_path_base) {
-                $bimage  = $ipprc->filename($name, $backup_path_base);
-                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
-                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
-                # bills 2011-01-24
-                # don't clean up the uncensored sources file
-                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
-            }
-            if ($recovery_path_base) {
-                $rimage  = $ipprc->filename($name, $recovery_path_base);
-                $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
-                $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
-                $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
-            }
-            # undef is not necessary, it's just to keep the lists in the same order.
-            delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources);
+            delete_files($bimage, $bmask, $bweight, $bsources);
         }
         my $command = "$magicdstool -tocleanedfile -magic_ds_id $magic_ds_id -component $component";

branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl

@@ -1266 +1266 @@
 }
 
+sub multi_date_verify_uniqueness_diff {
+    my $warp_id_1 = shift;
+    my $warp_id_2 = shift;
+    my $date = shift;
+    my $target = shift;
+
+    my $db = init_gpc_db();
+    $date =~ s/-//g;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $count = 0;
+    
+    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
+    my $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
+    $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    return($count);
+}
+
 sub pre_diff_queue {
     my $date = shift;
@@ -1375 +1398 @@
         $metadata_out{nsDiffState} = 'DIFFING';
     }
-
+    if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') {
+        foreach my $target (sort (keys %science_config)) {
+            if ($science_config{$target}{DIFFABLE} == 1) {
+                foreach my $filter (@filter_list) {
+                    multi_date_diff_queue($date,$target,$filter,$pretend);
+                }
+            }
+        }
+    }
 #     if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) {
 #       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
@@ -1511 +1542 @@
 
 }
+
+sub multi_date_diff_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $db = init_gpc_db();
+
+    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+
+    my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
+    
+    foreach my $object_row (@{ $object_ref }) {
+        my $this_object = shift @{ $object_row };
+#       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+#       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+#       $input_sth .= " ORDER BY dateobs ";
+        
+        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
+        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
+        $input_sth .=   " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+        $input_sth .=   " ORDER BY dateobs ";
+
+        my $warps = $db->selectall_arrayref( $input_sth );
+
+        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+        my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            $comment_hash{$this_comment} = $this_exp_id;
+        }
+        
+        if (($#{ $warps } + 1) % 2 != 0) {
+            print STDERR "md_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
+            if ($#{ $warps} + 1 == 1) {
+                print STDERR ": I can do no diffs with only one exposure.\n";
+                next;
+            }
+            else {
+                print STDERR ": I should declare an exposure to be faulty.\n";
+                my @keep_warps = ();
+#               print "@{ $warps }\n";
+                foreach my $this_warp (@{ $warps }) {
+                    my $this_comment = ${ $this_warp }[3];
+                    my $this_exp_id  = ${ $this_warp }[0];
+                    if ($comment_hash{$this_comment} == $this_exp_id) {
+                        push @keep_warps, $this_warp;
+                    }
+                    else {
+                        print STDERR "md_diff_queue: excluding $this_exp_id for $this_object\n";
+                    }
+                }
+                @{ $warps } = @keep_warps;
+#               print "@{ $warps }\n";
+            }
+        }
+        
+        while ($#{ $warps } > -1) {
+            my $input_warp = shift @{ $warps };
+            my $input_exp_id = ${ $input_warp }[0];
+            my $input_comment = ${ $input_warp }[3];
+
+            
+            my $template_warp = shift @{ $warps };
+
+            my $template_exp_id = ${ $template_warp }[0];
+            
+            my $input_warp_id = ${ $input_warp }[1];
+            my $template_warp_id = ${ $template_warp }[1];
+
+            my $input_warp_state = ${ $input_warp }[4];
+            my $template_warp_state = ${ $template_warp }[4];
+            
+            unless(defined($template_warp)&& defined($template_exp_id)) {
+                print STDERR "md_diff received an undef! $input_exp_id $input_comment $this_object T: $template_warp V: @$template_warp\n";
+                next;
+            }
+            $Npotential++;
+            
+            unless (defined($input_warp_id) && defined($template_warp_id) &&
+                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
+                print STDERR "md_Diff for this $date $target $input_exp_id ($input_warp_id $input_warp_state) $template_exp_id ($template_warp_id $template_warp_state) not fully processed\n";
+                next;
+            }
+
+            if (multi_date_verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
+                $Nqueued++;
+                print STDERR "md_Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
+                next;
+            }
+
+            my $new_data_group = "${data_group}.multi";
+            my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
+            $cmd .= "-input_label $label  -template_label $label ";
+            $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+            $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $new_data_group ";
+            $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+#               $cmd .= " -pretend ";
+            if (defined($pretend)) {
+                $cmd .= ' -pretend ';
+            }
+            if ($debug == 1) {
+                $cmd .= ' -pretend ';
+                print STDERR "md_Diffs would like to queue for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
+                print STDERR "md_diff_queue: $cmd\n";
+                print STDERR " $input_warp_id $template_warp_id\n";
+            }
+            
+            if (($debug == 0)&&(!defined($pretend))) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run ( command => $cmd, verbose => $verbose );
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                }
+                $Nqueued++;
+            }
+        }
+    }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#      }        
+
+}
             
 

branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl

@@ -255 +255 @@
         if (exists($nightlyscience_config{$target}{OBSMODE})) {
             $possible++;
-            if ($obsmode =~ /$nightlyscience_config{$target}{OBSMODE}/) {
+            if ($obsmode =~ /^$nightlyscience_config{$target}{OBSMODE}$/) {
                 $match++;
             }
@@ -261 +261 @@
         if (exists($nightlyscience_config{$target}{OBJECT})) {
             $possible++;
-            if ($object =~ /$nightlyscience_config{$target}{OBJECT}/) {
+            if ($object =~ /^$nightlyscience_config{$target}{OBJECT}$/) {
                 $match++;
             }
@@ -267 +267 @@
         if (exists($nightlyscience_config{$target}{COMMENT})) {
             $possible++;
-            if ($comment =~ /$nightlyscience_config{$target}{COMMENT}/) {
+            if ($comment =~ /^$nightlyscience_config{$target}{COMMENT}$/) {
                 $match++;
             }

branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl

@@ -71 +71 @@
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
-my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR );
-$logDest .= ".update" if ($run_state eq 'update');
+my ($logDest, $traceDest);
+if ($run_state eq 'new') {
+    $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0);
+    $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
+} elsif ($run_state eq 'update')  {
+    $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 0);
+    $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1);
+} else {
+    &my_die( "invalid run_state: $run_state", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR );
+}
+
+my $neb;
+my $scheme = file_scheme($outroot);
+if ($scheme and $scheme eq 'neb') {
+    $neb = $ipprc->nebulous();
+}
 
 $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
@@ -117 +131 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code);
+        &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code);
     }
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
     $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
 }
@@ -141 +155 @@
 
 
-my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id );
-my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id);
-my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id);
-my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id);
-my $outputPSF = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $outroot);
-my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id );
-my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id );
-my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id );
-my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id);
-my $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id);
-
-if ($run_state eq 'update') {
-    $traceDest .= ".update";
-    $outputStats .= ".update";
-}
-
-my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
+my $outputImage = prepare_output ("PSWARP.OUTPUT", $outroot, $skycell_id, 1);
+my $outputMask = prepare_output ("PSWARP.OUTPUT.MASK", $outroot, $skycell_id, 1);
+my $outputWeight = prepare_output ("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id, 1);
+my $outputSources = prepare_output ("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id, 1);
+my $outputPSF = prepare_output ("PSPHOT.PSF.SKY.SAVE", $outroot, 1);
+my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1);
+my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1);
+my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1);
+my $configuration;
+
+my $dump_config = 1;
+if ($run_state eq 'new') {
+    $configuration =  prepare_output ("PSWARP.CONFIG", $outroot, $skycell_id, 1);
+} else {
+    $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id) or
+        &my_die("Missing entry from camera config PSWARP.CONFIG", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    if ($ipprc->file_exists($configuration)) {
+        $dump_config = 0;
+    } else {
+        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
+
+        # XXX: should we create a new config dump file?
+        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
+        my $gone = 0;
+        if (storage_object_exists($configuration, \$gone)) {
+            if ($gone) {
+                $configuration = prepare_output('PSWARP.CONFIG', $outroot, $skycell_id, 1);
+                # if we dump the config we need to insure that the config dump represents
+                # the full processing
+            } else {
+                # file is temporarily not available. Don't dump config.
+                $dump_config = 0;
+            }
+        }
+    }
+}
+
+
+my $skyFile = prepare_output ("SKYCELL.TEMPLATE", $outroot, $skycell_id, 1);
 $ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
 ## XXX this seems to have insufficient error checking: dvoImageExtract can fail to write and still return a valid exit status
@@ -171 +207 @@
 foreach my $imfile (@$imfiles) {
     my $image = $imfile->{uri}; # Image name
-    my $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
+    my $weight = $ipprc->filename ("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
 
     my $mask;                   # Mask name
     if ($dynamicMasks) {
-        $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
+        $mask = $ipprc->filename ("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
     } else {
-        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id});
+        $mask = $ipprc->filename ("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id});
     }
 
@@ -186 +222 @@
     # Astrometry file: astrometry is done at the camera stage, and always results in a MEF file
     # XXX allow an option to use the image header astrometry?
-    my $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base});
+    my $astrom = $ipprc->filename ($astromSource, $imfile->{cam_path_base});
 
     &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom);
@@ -245 +281 @@
         $do_stats = 1;
     } else {
-        #$command .= " -ipprc $configuration";
-        my $resolved = $ipprc->file_resolve($configuration);
-        $command .= " -ipprc $resolved";
+        $command .= " -ipprc $configuration";
     }
     if ($do_stats) {
@@ -263 +297 @@
     if ($do_stats) {
         # Check first for the stats file
+        check_output($outputStats, 0);
         my $outputStatsReal = $ipprc->file_resolve($outputStats);
-        &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
+#        &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
         &my_die("Stats file has zero size: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -s $outputStatsReal;
 
-        # measure chip stats
+        # measure skycell stats
         $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL";
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -283 +318 @@
 
         if (!$quality) {
-            &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
-            &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
-            &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
-            &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
-            &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF);
-            if ($run_state eq 'new') {
-                &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
+        #    &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
+        #    &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
+        #    &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
+        #    &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
+        #    &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF);
+
+            check_output($outputImage, 0);
+            check_output($outputMask, 0);
+            check_output($outputWeight, 0);
+            check_output($outputSources, 1);
+            check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF')  ;
+            if ($dump_config)  {
+                check_output($configuration, 1);
+#                &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
             }
         }
@@ -335 +377 @@
     }
 }
+
+exit 0;
+
+# Prepare to write to an output file
+#   Lookup the filename in the rules.
+#   Make sure that if file exists and is a nebulous file that there is only one instance
+#   Deal with files that have been lost.
+sub prepare_output
+{
+    my $filerule = shift;
+    my $outroot  = shift;
+    my $skycell_id = shift;
+    my $delete = shift;
+    $delete = 0 if !defined $delete;
+
+    my $error;
+    my $output = $ipprc->prepare_output($filerule, $outroot, $skycell_id, $delete, \$error)
+                    or &my_die("failed to prepare output file for: $filerule", $warp_id, $skycell_id, $tess_dir, $error);
+    return $output;
+}
+
+sub check_output
+{
+    my $file = shift;
+    my $replicate = shift;
+
+    if (!defined $file) {
+        return;
+    }
+
+    &my_die("Couldn't find expected output file: $file",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
+
+    if ($replicate and $neb) {
+        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
+    }
+}
+
+# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that
+# does not exist and one that all of the instances have been lost.
+# XXXX This should be implemented properly in Nebulous
+# For now uses Bill's script 'whichnode' which queries the nebulous database directly
+
+my $whichnode;
+sub storage_object_exists
+{
+    return 0 if !$neb;
+
+    my $file = shift;
+    my $ref_all_gone = shift;
+
+    my $exists = $neb->storage_object_exists($file);
+    if (!$exists) {
+        return 0;
+    }
+
+    if (!$whichnode) {
+        $whichnode = can_run('whichnode') or
+            &my_die("Can't find whichnode",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my $command = "$whichnode $file";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform whichnode: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my @lines = split "\n", (join "", @$stdout_buf);
+
+    if (scalar @lines == 0) {
+        # no output the file is really and truely gone
+        # XXX: this is now caught above
+        print STDERR "storage object for $file does not exist\n";
+        return 0;
+    }
+
+    my $numGone = 0;
+    my $numNotGone = 0;
+    foreach my $line (@lines) {
+        chomp $line;
+
+        # output lines are either
+        #   "volume available"
+        # or 
+        #   "volume not available"
+
+        my ($volume, $answer, undef) = split " ", $line;
+        # our hack is if the volume has an X in the name it's gone
+        if ($volume =~ /X/) {
+            print STDERR "$file is on $volume which is gone\n";
+            $numGone++;
+        } elsif ($answer eq 'available') {
+            $numNotGone++;
+        } elsif ($answer eq 'not') {
+            print STDERR "$file is on $volume which is not available\n";
+            $numNotGone++;
+        } else {
+            print STDERR "unexpected output from whichnode: $line\n";
+        }
+    }
+    # if there are any instances that are not on a gone volume set all_gone to 0
+    if ($numNotGone == 0 and $numGone > 0) {
+        $$ref_all_gone = 1;
+    } else {
+        $$ref_all_gone = 0;
+    }
+
+    # storage object exists so return true
+    return 1;
+}
+
 
 sub my_die