Index: trunk/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- trunk/ippScripts/scripts/ipp_cleanup.pl	(revision 35465)
+++ trunk/ippScripts/scripts/ipp_cleanup.pl	(revision 35787)
@@ -202,5 +202,7 @@
                 print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf;
                 print STDERR "\nSTDERR:\n" . join "", @$stderr_buf;
-                &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code);
+                # smf file is probably corrupt. Carry on. We will not clean sources unless another good smf is found
+                # &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code);
+                next;
             }
             $good_smf++;
@@ -374,5 +376,5 @@
     # the imfiles being in the cleaned state.
     # Due to missing files we don't always clean individual components.
-    # unless there was an error set state to $done_state
+    # So unless there was an error set state to $done_state
     if (!$set_error_state_for_run or $num_errors eq 0) {
         # set state to $done_state
@@ -1916,6 +1918,6 @@
     }
 
-    # XXX: Is not having any files to process really a bug?
-    if (@$stdout_buf == 0)  {
+    # XXX: Is not having any files to process really a bug? NO
+    if (0 and @$stdout_buf == 0)  {
         my $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state";
         $command .= " -dbname $dbname" if defined $dbname;
@@ -1935,4 +1937,6 @@
 
     # loop over all of the imfiles, determine the path_base and class_id for each
+    my $num_errors = 0;
+    my $num_updated = 0;
     foreach my $imfile (@$imfiles) {
         my $class_id = $imfile->{class_id};
@@ -1989,4 +1993,5 @@
 
         if ($status)  {
+            $num_updated++;
             my $command = "$bgtool -chip_bg_id $stage_id -class_id $class_id";
             if ($mode eq "goto_cleaned") {
@@ -2010,29 +2015,49 @@
 
         } else {
-
-            # if an error happens for one chip, the chipBackgroundRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
-            my $command = "$bgtool -updatechipimfile -chip_bg_id $stage_id -class_id $class_id -set_data_state $error_state";
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            $num_errors++;
+
+            if ($set_error_state_for_file) {
+                # if an error happens for one chip, the chipBackgroundRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
+                my $command = "$bgtool -updatechipimfile -chip_bg_id $stage_id -class_id $class_id -set_data_state $error_state";
+                $command .= " -dbname $dbname" if defined $dbname;
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code);
+                }
+            }
+            if ($set_error_state_for_run) {
+
+                # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
+                $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state";
+                $command .= " -dbname $dbname" if defined $dbname;
+
+                ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code);
-            }
-
-            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
-            $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state";
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
-            }
-
-        }
-    }
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+                }
+
+            }
+        }
+    }
+    if (!$set_error_state_for_run or ($num_errors eq 0)) {
+        # Set run to done state
+        my $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $done_state";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+        }
+    }
+    print "Cleanup completed for chip_bg_id $stage_id. num_updated: $num_updated";
+    print " num_errors:  $num_errors" if $num_errors;
+    print "\n";
     exit 0;
 }
@@ -2060,6 +2085,6 @@
     }
 
-    # XXX: Is not having any files to process really a bug?
-    if (@$stdout_buf == 0)  {
+    # XXX: Is not having any files to process really a bug? NO
+    if (0 and @$stdout_buf == 0)  {
         my $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state";
         $command .= " -dbname $dbname" if defined $dbname;
@@ -2079,4 +2104,6 @@
 
     # loop over all of the imfiles, determine the path_base and skycell_id for each
+    my $num_errors = 0;
+    my $num_updated = 0;
     foreach my $imfile (@$imfiles) {
         my $skycell_id = $imfile->{skycell_id};
@@ -2096,4 +2123,6 @@
                         print STDERR "forcing cleanup for warpBackgroundSkyfile $stage_id $skycell_id "
                             . " because config file ($config_file) is gone\n";
+                    } elsif ($imfile->{fault}) {
+                        print STDERR "forcing cleanup for faulted warpBackgroundSkyfile\n";
                     } else {
                         print STDERR "skipping cleanup for warpBackgroundSkyfile $stage_id $skycell_id "
@@ -2103,13 +2132,4 @@
                 }
             }
-            elsif ($mode eq "goto_scrubbed") {
-                my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
-
-                if ($ipprc->file_exists($config_file)) {
-                    print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $skycell_id "
-                        . " because config file ($config_file) is present\n";
-                    $status = 0;
-                }
-            }
         }
 
@@ -2133,4 +2153,5 @@
 
         if ($status)  {
+            $num_updated++;
             my $command = "$bgtool -warp_bg_id $stage_id -skycell_id $skycell_id";
             # recall that only goto_cleaned is supported (currently)
@@ -2156,27 +2177,50 @@
         } else {
 
-            # if an error happens for one skycell, the warpBackgroundRun will stay in goto_*, but the skycells will go to error_* (matching the goto_*)
-            my $command = "$bgtool -updatewarpskyfile -warp_bg_id $stage_id -skycell_id $skycell_id -set_state $error_state";
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+          $num_errors++;
+
+          # NOTE: bgtool does not support -updatewarpskyfile
+            if (0) {
+                # if an error happens for one skycell, the warpBackgroundRun will stay in goto_*, but the skycells will go to error_* (matching the goto_*)
+                my $command = "$bgtool -updatewarpskyfile -warp_bg_id $stage_id -skycell_id $skycell_id -set_state $error_state";
+                $command .= " -dbname $dbname" if defined $dbname;
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code);
+                }
+            }
+
+            if ($set_error_state_for_run) {
+
+                # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
+                $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state";
+                $command .= " -dbname $dbname" if defined $dbname;
+
+                ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code);
-            }
-
-            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
-            $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state";
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code);
-            }
-        }
-    }
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code);
+                }
+            }
+        }
+    }
+    if (($num_errors == 0) || !$set_error_state_for_run) {
+        # Set run to done state
+        my $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $done_state";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+        }
+    }
+    print "Cleanup completed for warp_bg_id $stage_id. num_updated: $num_updated";
+    print " num_errors:  $num_errors" if $num_errors;
+    print "\n";
     exit 0;
 }