Index: /tags/ipp-20101215/PS-IPP-Config/lib/PS/IPP/Config.pm
===================================================================
--- /tags/ipp-20101215/PS-IPP-Config/lib/PS/IPP/Config.pm	(revision 30634)
+++ /tags/ipp-20101215/PS-IPP-Config/lib/PS/IPP/Config.pm	(revision 30635)
@@ -1148,43 +1148,10 @@
     if ($neb->storage_object_exists($output)) {
         if ($delete_existing) {
-            # avoid dead instances by moving the file before deleting it.
-            # append current time to form new name
-            my $todelete;
-            eval {
-                # parse the key so that we can compute the new name
-                require Nebulous::Key;
-            };
-            if ($@) {
-                carp "Can't find Nebulous::Key";
-                $$r_error = $PS_EXIT_CONFIG_ERROR;
-                return undef;
-            }
-            eval {
-                # parse the key so that we can compute the new name
-                my $neb_key = Nebulous::Key::parse_neb_key($output);
-                die "parse_neb_key failed" if !$neb_key;
-                my $path = $neb_key->path;
-                die "neb_key has no path" if !$path;
-                my $ticks = time();
-                $todelete = "ipp_trash/$path.$ticks";
-                $neb->move($output, $todelete);
-            };
-            if ($@) {
-                carp "nebulous move failed for $output";
+            $$r_error = $self->_kill_nebulous_file($output);
+            if ($$r_error) {
                 $output = undef;
-                $$r_error = $PS_EXIT_SYS_ERROR;
-            }
-            if ($todelete) {
-                eval {
-                    $neb->delete($todelete);
-                };
-                if ($@) {
-                    carp "nebulous delete for $todelete failed. Ignoring.\n";
-                    $$r_error = $PS_EXIT_SYS_ERROR;
-                }
             }
         } else {
             # Make sure that there is only 1 instance.
-
             eval {
                 $neb->there_can_be_only_one($output);
@@ -1198,4 +1165,77 @@
     }
     return $output;
+}
+
+# _kill_nebulous_file: reliably get a nebulous file out of way.
+# First move it to the trash then attempt to delete it. No failure if delete fails.
+# The file is in the trash.
+# Assumes that file is a nebulous file with a storage object that exists
+# Users should call kill_file()
+# Returns 0 on success otherwise and a PS_EXIT error code on failure
+sub _kill_nebulous_file {
+    my $self = shift;
+    my $filename = shift;
+    my $neb = $self->nebulous;
+
+    # avoid dead instances by moving the file before deleting it.
+    # append current time to form new name
+    my $todelete;
+    eval {
+        # parse the key so that we can compute the new name
+        require Nebulous::Key;
+    };
+    if ($@) {
+        carp "Can't find Nebulous::Key";
+        return $PS_EXIT_CONFIG_ERROR;
+    }
+    eval {
+        # parse the key so that we can compute the new name
+        my $neb_key = Nebulous::Key::parse_neb_key($filename);
+        die "parse_neb_key failed" if !$neb_key;
+        my $path = $neb_key->path;
+        die "neb_key has no path" if !$path;
+        my $ticks = time();
+        $todelete = "ipp_trash/$path.$ticks";
+        $neb->move($filename, $todelete);
+    };
+    if ($@) {
+        carp "nebulous move failed for $filename";
+        return $PS_EXIT_SYS_ERROR;
+    }
+    if ($todelete) {
+        eval {
+            $neb->delete($todelete);
+        };
+        if ($@) {
+            carp "nebulous delete for $todelete failed. Ignoring.\n";
+            return $PS_EXIT_SYS_ERROR;
+        }
+    }
+    return 0;
+}
+
+# user level interface to kill_file reliably get a file out of way.
+#
+# If a nebulous file, move it to the trash then attempt to delete it. 
+# if non nebulous file just delete it.
+# Returns 0 on success otherwise and a PS_EXIT error code on failure
+
+sub kill_file {
+    my $self = shift;
+    my $filename = shift;
+    my $neb = $self->nebulous;
+
+    my $scheme = file_scheme($filename);
+    if (!$scheme or ($scheme ne 'neb')) {
+        if ($self->file_exists($filename)) {
+            if (!$self->file_delete($filename)) {
+                carp "failed to delete $filename";
+                return $PS_EXIT_SYS_ERROR;
+            }
+        }
+    } elsif ($neb->storage_object_exists($filename)) {
+        $self->_kill_nebulous_file($filename);
+    }
+    return 0;
 }
 
Index: /tags/ipp-20101215/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- /tags/ipp-20101215/ippScripts/scripts/ipp_cleanup.pl	(revision 30634)
+++ /tags/ipp-20101215/ippScripts/scripts/ipp_cleanup.pl	(revision 30635)
@@ -207,14 +207,5 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -324,14 +315,5 @@
         }
 
-        # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-        my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-        $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-        $command .= " -dbname $dbname" if defined $dbname;
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-        }
+        set_destreak_goto_cleaned();
 
     } else {
@@ -469,14 +451,5 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
          } else {
@@ -603,14 +576,5 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -759,14 +723,5 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -1945,36 +1900,12 @@
 {
     my $files = shift; # reference to a list of files to unlink
-#     my $test_verbose = 1;
-
-#     if ($test_verbose == 1) {
-#       open(TMPLOG,">>/tmp/czw.cleanup.log");
-#       flock(TMPLOG,2);
-#     }
-
-    # this script is, of course, very dangerous.
+
     foreach my $file (@$files) {
         print STDERR "unlinking $stage $stage_id $file";
-        unless ($ipprc->file_exists($file)) {
-            print STDERR "\t File not found\n";
-        }
-        else {
-            print STDERR "\n";
-        }
-
-#       if ($test_verbose == 1) {
-#           print TMPLOG "$stage $stage_id $file";
-
-#           else {
-#               print TMPLOG "\n";
-#           }
-#       }
-
-        $ipprc->file_delete($file);
-    }
-
-#     if ($test_verbose == 1) {
-#       flock(TMPLOG,8);
-#       close(TMPLOG);
-#     }
+
+        my $error_code = $ipprc->kill_file($file);
+
+        &my_die("failed to kill $file", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if $error_code;
+    }
 
     return 1;
@@ -2062,4 +1993,23 @@
 }
 
+# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
+my $ds_done = 0;
+sub set_destreak_goto_cleaned {
+
+    return if $ds_done;
+
+    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
+    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
+    my $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    $ds_done = 1;
+}
+
 # XXX we currently do not set the error state in the db on my_die
 sub my_die
Index: /tags/ipp-20101215/ippScripts/scripts/magic_destreak_cleanup.pl
===================================================================
--- /tags/ipp-20101215/ippScripts/scripts/magic_destreak_cleanup.pl	(revision 30634)
+++ /tags/ipp-20101215/ippScripts/scripts/magic_destreak_cleanup.pl	(revision 30635)
@@ -44,5 +44,5 @@
            'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
            'camera=s'       => \$camera,     # camera for evaluating file rules
-           'stage=s'        => \$stage,     # camera for evaluating file rules
+           'stage=s'        => \$stage,      # ipp stage for this magicDSRun
            'save-temps'     => \$save_temps, # Save temporary files?
            'dbname=s'       => \$dbname,     # Database name
@@ -160,5 +160,5 @@
         my $backup_path_base = $comp->{backup_path_base};
         my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
-        my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
+#        my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
 
         if ($stage eq "chip") {
@@ -219,5 +219,5 @@
         }
 
-        delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
+        delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask);
 
         if ($stage eq "diff" and $warp_warp) {
@@ -252,14 +252,6 @@
     foreach my $file (@_) {
         if ($file) {
-            if ($ipprc->file_exists($file)) {
-                if (!$no_update) {
-                    print STDERR "deleting $file\n" if $verbose;
-                    $ipprc->file_delete($file, 1) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR);
-                    } else {
-                    print STDERR "skipping delete $file\n";
-                }
-            } else {
-                print STDERR "$file not found\n" if $verbose;
-            }
+            my $error_code = $ipprc->kill_file($file);
+            my_die("Failed to delete $file", $magic_ds_id, $error_code) if $error_code;
         }
     }