Index: trunk/ippScripts/scripts/magic_destreak.pl
===================================================================
--- trunk/ippScripts/scripts/magic_destreak.pl	(revision 24723)
+++ trunk/ippScripts/scripts/magic_destreak.pl	(revision 24726)
@@ -21,4 +21,5 @@
 
 use PS::IPP::Config 1.01 qw( :standard );
+use Nebulous::Client;
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -30,7 +31,5 @@
 my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1);
 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
-# XXX: replace with real name for tool
-my $censordetections = "";
-# my $censordetections = can_run('censordetections') or (warn "Can't find censordetections" and $missing_tools = 1);
+my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -54,5 +53,5 @@
            'path_base=s'    => \$path_base,  # path_base of the input
            'inverse'        => \$inverse,    # Inverse subtraction?
-           'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
+           'cam_path_base=s'=> \$cam_path_base,  # path_base from camera stage (for chip and raw)
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
            'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
@@ -102,8 +101,23 @@
 my $basename = basename($path_base);
 my $nebulousInput = inNebulous($dirname);
+my $nebulousOutput = inNebulous($outroot);
 
 # parse replace arguments check for errors and set up
 # the appropriate paths
+my $nebulous;
 if (defined($replace) and ($replace eq "T")) {
+    # for camera stage we need a handle to the nebulous server
+    if ($stage eq 'camera') {
+        my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
+        &my_die("cannot find NEB_SERVER in site configuration", 
+                                        $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) 
+            if !$ nebulousServer;
+
+        $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); };
+        if ($@ or not defined $nebulous) {
+            &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer",
+                                    $magic_ds_id, $component , $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
     $replace = 1;
 } else {
@@ -111,4 +125,5 @@
 }
 
+
 # default value is "NULL" do not use
 if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
@@ -122,5 +137,5 @@
 
 # create the output directories if it is not a nebulous path and it doesn't exist
-if (!inNebulous($outroot)) {
+if (! $nebulousOutput) {
     if (! -e $outroot ) {
         my $code = system "mkdir -p $outroot";
@@ -128,9 +143,4 @@
                 $code >> 8) if $code;
     }
-#} elsif ($replace) {
-    # if input is in nebulous and we've been asked to replace the input with the destreaked
-    # version the output must be in nebulous as well
-    # XXX Is this necessary? if ($replace) the outputs are going to the nebulous "directory" of the
-    # input anyways. Why require that other files (log) be in nebulous
 }
 
@@ -208,4 +218,14 @@
         $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
         $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
+        if (!$ipprc->file_exists($mask)) {
+            carp("camera mask file $mask for $component not found. Continuing using mask from chip stage.");
+            $mask = $ch_mask;
+            $ch_mask = undef;
+        }
+        # XXX: should we censor the sources as well? For now we're leaving them out of the distribution bundles
+        # because they confuse people
+        # XXX: make sure that this is the right file rule
+        # $sources = $ipprc->filename("PSPHOT.OUT.CMF.SPL",  $path_base);
+
     } elsif ($stage eq "warp") {
         $image  = $ipprc->filename("PSWARP.OUTPUT", $path_base);
@@ -247,4 +267,6 @@
     }
 } else {
+    # camera stage. The only work to do is to censor the detections file
+
     my $tempOutRoot = "/tmp/destreak";
     my ($maskListFile,   $maskListName)   = tempfile( "$tempOutRoot.mask.list.XXXX",   UNLINK => !$save_temps);
@@ -272,4 +294,8 @@
         if (!$quality) {
             my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id); 
+            if (! $ipprc->file_exists($mask)) {
+                # camera mask doesn't exist for this chip. Fall back to the chip mask
+                $mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $file->{path_base}, $class_id);
+            }
             print $maskListFile "$mask\n";
         }
@@ -279,9 +305,6 @@
     my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $path_base);
     {
-        my $output = $tmproot . basename($astrom);
-        my $command = "$censordetections -stage $stage -output $output";
-        $command .= " -astrom $astrom";
-        $command .= " -masklist $maskListName";
-        $command .= " -replace" if $replace;
+        my $command = "$censorObjects -file $astrom -masklist $maskListName $backup_path_base";
+        # $command .= " -replace" if $replace;
         $command .= " -dbname $dbname" if defined $dbname;
         unless (defined $no_op) {
@@ -290,5 +313,13 @@
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
+                &my_die("Unable to perform censorObjects: $error_code", $magic_ds_id, $component, $error_code);
+            }
+            my $output = $ipprc->filename("CENSOR.OUTPUT", $backup_path_base); 
+            &my_die("expected output file $output not found ", $magic_ds_id, $component, $PS_EXIT_DATA_ERROR)
+                unless $ipprc->file_exists($output);
+
+            if ($replace) {
+                $nebulous->swap($astrom, $output) or
+                    &my_die("nebulous swap failed $astrom $output", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
             }
         } else {
Index: trunk/ippScripts/scripts/magic_destreak_revert.pl
===================================================================
--- trunk/ippScripts/scripts/magic_destreak_revert.pl	(revision 24723)
+++ trunk/ippScripts/scripts/magic_destreak_revert.pl	(revision 24726)
@@ -70,7 +70,4 @@
     defined $outroot;
 
-# XXX: for testing
-$no_update = 1;
-
 &my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'raw' and (!$bytes or !$md5sum));
 
@@ -87,5 +84,4 @@
 if ($@ or not defined $nebulous) {
     &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
-    return undef;
 }
 
@@ -109,9 +105,4 @@
 my $nebulousInput = inNebulous($dirname);
 
-# parse replace arguments check for errors and set up
-# the appropriate paths
-# this program doesn't make sense if $replace is false
-&my_die("Invalid value for replace: $replace", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if $replace ne "T";
-
 if (defined($replace) and ($replace eq "T")) {
     $replace = 1;
@@ -120,5 +111,9 @@
 }
 
-# default value is "NULL" do not use
+&my_die("replace not allowed for non-nebulous files", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) 
+    if ($replace eq "T") and !$nebulousInput;
+
+
+# default value is "NULL" do not use it
 if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
     $recoveryroot = undef;
@@ -135,5 +130,5 @@
     $backup_path_base = $tmproot . "$basename";
 } else {
-    # note: trailing / is necessary
+    # note: the trailing / here is necessary
     $tmproot = "$outroot/";
     $backup_path_base = $tmproot . $basename;
@@ -189,40 +184,44 @@
 }
 
-do_swap($image, $bimage) or
+revert_file($image, $bimage) or
         &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+
+# don't commit next the next couple of lines and see xxx below
+# change to if 0 to revert chip runs without masks
+if (0) {
+
 if ($mask) {
-    my $swap_result = do_swap($mask, $bmask);
-    if (! $swap_result) {
-        # delete next 5 lines
-        if ($stage eq "chip") {
-            # XXX: this is a kludge to get around my test environment
-            # Don't commit
-            print STDERR "failed to resotre camera mask file, proceeding\n";
+    if (!revert_file($mask, $bmask)) {
+        if ($stage eq 'chip') {
+            # XXX: don't fail if the mask file fails to revert. It probably doesn't exist
+            # Handle this properly
+            print STDERR "failure to revert mask file, ignored\n";
         } else {
-        &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
-        }  # delete this line
-    }
-}
+            &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+        }
+    }
+}
+
 if ($ch_mask) {
-    do_swap($ch_mask, $bch_mask) or
-        &my_die("failed to restore chip mask", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+    # Handle this properly
+    if (!revert_file($ch_mask, $bch_mask)) {
+        # print STDERR "failure to revert chip mask file, ignored\n";
+        &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+    }
 } 
+
+# xxx don't commmit next line
+}
+
 if ($weight) {
-    do_swap($weight, $bweight) or
+    revert_file($weight, $bweight) or
         &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
 }
 
-# now reverse the desreaked file
-# Change magicdstool to set the magicked value to zero
-# Actually we should probably do this first to insure that no data gets deleted
-# Input result into database
-if (0) {
-    # XXX: this is the wrong mode
-    my $command = "$magicdstool -adddestreakedfile";
+# now revert the row in the database
+{
+    my $command = "$magicdstool -revertdestreakedfile";
     $command   .= " -magic_ds_id $magic_ds_id";
     $command   .= " -component $component";
-    $command   .= " -setmagicked" if $replace;
-    $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
-    $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
     $command   .= " -dbname $dbname" if defined $dbname;
 
@@ -242,5 +241,4 @@
 
 
-
 ### Pau.
 
@@ -256,5 +254,5 @@
 }
 
-sub do_swap
+sub revert_file
 {
     my $original = shift;
@@ -297,5 +295,5 @@
         if ($b_path and -e $b_path) {
             print "ready to delete backup\n" if $verbose;
-            if (! $nebulous->delete($backup)) {
+            if (! $ipprc->file_delete($backup)) {
                 print "failed to delete $backup\n";
                 return 0;
@@ -307,5 +305,5 @@
         # delete the 'backup' (destreaked target) file if it exists
         if ($b_path and -e $b_path) {
-            if (! $nebulous->delete($backup)) {
+            if (! $ipprc->file_delete($backup)) {
                 print "failed to delete $backup\n";
                 return 0;
@@ -348,20 +346,21 @@
     $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
 
-    my $command = "$magicdstool -adddestreakedfile";
-    $command   .= " -magic_ds_id $magic_ds_id";
-    $command   .= " -component $component";
-    $command   .= " -fault $exit_code";
-    $command   .= " -dbname $dbname" if defined $dbname;
-
-    # Add the processed file to the database
-    unless ($no_update) {
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            carp("failed to update database for $magic_ds_id $component");
-        }
-    } else {
-        print "Skipping command: $command\n";
-    }
+    # we wouldn't be here unless an entry already exists in the database
+#    my $command = "$magicdstool -adddestreakedfile";
+#    $command   .= " -magic_ds_id $magic_ds_id";
+#    $command   .= " -component $component";
+#    $command   .= " -fault $exit_code";
+#    $command   .= " -dbname $dbname" if defined $dbname;
+
+#    # Add the processed file to the database
+#    unless ($no_update) {
+#        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#            run(command => $command, verbose => $verbose);
+#        unless ($success) {
+#            carp("failed to update database for $magic_ds_id $component");
+#        }
+#    } else {
+#        print "Skipping command: $command\n";
+#    }
 
     carp($msg);