Index: trunk/ippScripts/scripts/magic_destreak_revert.pl
===================================================================
--- trunk/ippScripts/scripts/magic_destreak_revert.pl	(revision 24686)
+++ trunk/ippScripts/scripts/magic_destreak_revert.pl	(revision 24686)
@@ -0,0 +1,371 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename dirname );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+use Nebulous::Client;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $magicdstool   = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
+my $isdestreaked = can_run('isdestreaked') or (warn "Can't find isdestreaked" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base);
+my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum);
+my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
+
+GetOptions(
+           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,     # camera for evaluating file rules
+           'stage=s'        => \$stage,      # raw, chip, warp, or diff
+           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
+           'component=s'    => \$component,  # the class_id or skycell_id
+           'path_base=s'    => \$path_base,  # path_base of the input
+           'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
+           'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
+           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'replace=s'      => \$replace,    # replace the input images with the results.
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database?
+           'no-op'          => \$no_op,      # Don't do any operations?
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot",
+           -exitval => 3) unless
+    defined $magic_ds_id and
+    defined $camera and
+    defined $stage and
+    defined $stage_id and
+    defined $component and
+    defined $path_base and
+    defined $outroot;
+
+# XXX: for testing
+$no_update = 1;
+
+&my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'raw' and (!$bytes or !$md5sum));
+
+&my_die("cam_path_base is required for chip stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'chip' and !$cam_path_base);
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
+
+my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
+&my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer;
+
+my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); };
+if ($@ or not defined $nebulous) {
+    &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+    return undef;
+}
+
+
+my $class_id;
+my $skycell_id;
+if (($stage eq "raw") or ($stage eq "chip")) {
+    $class_id = $component;
+} elsif ($stage eq "warp") {
+    $skycell_id = $component;
+} elsif ($stage eq "diff") {
+    $skycell_id = $component;
+} else {
+    &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $dirname  = dirname($path_base);
+my $basename = basename($path_base);
+my $nebulousInput = inNebulous($dirname);
+
+# parse replace arguments check for errors and set up
+# the appropriate paths
+# this program doesn't make sense if $replace is false
+&my_die("Invalid value for replace: $replace", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if $replace ne "T";
+
+if (defined($replace) and ($replace eq "T")) {
+    $replace = 1;
+} else {
+    $replace = 0;
+}
+
+# default value is "NULL" do not use
+if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
+    $recoveryroot = undef;
+}
+
+my $backup_path_base;
+my $tmproot;
+if ($replace) {
+    # in replace mode, we place the output files in the same "directory" as the inputs
+    # Nebulous requires this for the two inputs to nebSwap which we use
+    # We prepend the path with SR_ This causes the filenames for instances of the swapped files to
+    # have SR in them.
+    $tmproot = $dirname . "/SR_";
+    $backup_path_base = $tmproot . "$basename";
+} else {
+    # note: trailing / is necessary
+    $tmproot = "$outroot/";
+    $backup_path_base = $tmproot . $basename;
+}
+
+my $recovery_path_base;
+if ($recoveryroot) {
+    # note: trailing / is necessary
+    $recovery_path_base = "$recoveryroot/$basename/";
+}
+
+
+my ($image, $mask, $ch_mask, $weight);
+my ($bimage, $bmask, $bch_mask, $bweight);
+
+if ($stage eq "raw") {
+    if ($path_base =~ /.*\.fits$/) {
+        # removal of uri from rawImfile hasn't happened yet
+        $image = $path_base;
+    } else {
+        # XXX: should have a file rule
+        $image = $path_base . ".fits";
+    }
+    $bimage = $backup_path_base . ".fits";
+} elsif ($stage eq "chip") {
+    # we use the mask output from the camera stage for input and replace
+    # the output of the chip stage with that mask as well.
+    $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
+    $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
+    $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
+    $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+
+    $bimage  = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id);
+    # This is kludgey but correct
+    $bmask   = dirname($backup_path_base) . "/SR_" . basename($mask);
+    $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id);
+    $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id);
+} elsif ($stage eq "warp") {
+    $image  = $ipprc->filename("PSWARP.OUTPUT", $path_base);
+    $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
+    $weight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
+    $bimage  = $ipprc->filename("PSWARP.OUTPUT", $backup_path_base);
+    $bmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $backup_path_base);
+    $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base);
+} elsif ($stage eq "diff") {
+    my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images
+    $image  = $ipprc->filename($name, $path_base);
+    $mask   = $ipprc->filename("$name.MASK", $path_base);
+    $weight = $ipprc->filename("$name.VARIANCE", $path_base);
+    $bimage  = $ipprc->filename($name, $backup_path_base);
+    $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
+    $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
+}
+
+do_swap($image, $bimage) or
+        &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+if ($mask) {
+    my $swap_result = do_swap($mask, $bmask);
+    if (! $swap_result) {
+        # delete next 5 lines
+        if ($stage eq "chip") {
+            # XXX: this is a kludge to get around my test environment
+            # Don't commit
+            print STDERR "failed to resotre camera mask file, proceeding\n";
+        } else {
+        &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+        }  # delete this line
+    }
+}
+if ($ch_mask) {
+    do_swap($ch_mask, $bch_mask) or
+        &my_die("failed to restore chip mask", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+} 
+if ($weight) {
+    do_swap($weight, $bweight) or
+        &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+# now reverse the desreaked file
+# Change magicdstool to set the magicked value to zero
+# Actually we should probably do this first to insure that no data gets deleted
+# Input result into database
+if (0) {
+    # XXX: this is the wrong mode
+    my $command = "$magicdstool -adddestreakedfile";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -component $component";
+    $command   .= " -setmagicked" if $replace;
+    $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
+    $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, $component,
+                $error_code);
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+}
+
+
+
+### Pau.
+
+sub check_keyword
+{
+    my $filename = shift;
+    my $command = "$isdestreaked $filename";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+
+    return $error_code >> 8;
+}
+
+sub do_swap
+{
+    my $original = shift;
+    my $backup   = shift;
+
+    my $o_path = $ipprc->file_resolve($original);
+    my $original_result;
+    if ($o_path) {
+        # invoke the program isdestreaked to check for the destreaked keyword
+        $original_result = check_keyword($o_path);
+    } else {
+        print STDERR "no instances found for $original\n";
+        $original_result = $PS_EXIT_DATA_ERROR;
+    }
+    my $original_is_destreaked = $original_result eq 0;
+    my $original_is_not_destreaked = $original_result eq 42;
+
+    my $b_path = $ipprc->file_resolve($backup);
+    my $backup_result;
+    if ($b_path) {
+        $backup_result = check_keyword($b_path);
+    } else {
+        print STDERR "no instances found for $backup\n";
+        $backup_result = $PS_EXIT_DATA_ERROR;
+    }
+    my $backup_is_destreaked = $backup_result eq 0;
+    my $backup_is_not_destreaked = $backup_result eq 42;
+
+    if ($backup_is_not_destreaked) {
+        # XXX TODO if stage is raw, check that backup has the correct size and md5sum
+
+        # Do we need to make this test? After the swap we're going to delete the file anyways
+        print "ready to swap $backup to $original\n" if $verbose;
+
+        if (! $nebulous->swap($backup, $original)) {
+            print "failed to swap $backup to $original\n";
+            return 0;
+        }
+
+        if ($b_path and -e $b_path) {
+            print "ready to delete backup\n" if $verbose;
+            if (! $nebulous->delete($backup)) {
+                print "failed to delete $backup\n";
+                return 0;
+            }
+        }
+
+    } elsif ($original_is_not_destreaked) {
+        print "original uri: $original is not a destreaked file no need to swap backup_result: $backup_result\n";
+        # delete the 'backup' (destreaked target) file if it exists
+        if ($b_path and -e $b_path) {
+            if (! $nebulous->delete($backup)) {
+                print "failed to delete $backup\n";
+                return 0;
+            }
+        }
+    } else {
+        print STDERR "Error: neither file is an un de-streaked file.\n";
+        print STDERR " backup key: $backup status: $backup_result\n";
+        print STDERR " original key: $original status $original_result\n";
+        return 0;
+    }
+    return 1;
+}
+
+
+sub file_check
+{
+    my $file = shift;           # Name of file
+    &my_die("Unable to find output file: $file", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless
+        $ipprc->file_exists($file);
+}
+
+sub inNebulous
+{
+    my $path = shift;
+
+    my $scheme = file_scheme($path);
+
+    return $scheme and ($scheme eq "neb");
+}
+
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $magic_ds_id = shift;    # Magic DS identifier
+    my $component = shift;      # class_id or skycell_id
+    my $exit_code = shift;      # Exit code to add
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    my $command = "$magicdstool -adddestreakedfile";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -component $component";
+    $command   .= " -fault $exit_code";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $magic_ds_id $component");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
+    carp($msg);
+    exit $exit_code;
+}
+
+__END__