Index: trunk/ippScripts/scripts/destreak_restore_camera.pl
===================================================================
--- trunk/ippScripts/scripts/destreak_restore_camera.pl	(revision 33602)
+++ trunk/ippScripts/scripts/destreak_restore_camera.pl	(revision 33602)
@@ -0,0 +1,453 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use Carp;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+my $date = `date`;
+print STDERR "\n\n";
+print STDERR "Starting script $0 on $host at $date\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use Digest::MD5::File qw( file_md5_hex );
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename dirname );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+use Nebulous::Client;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $magicdstool   = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
+my $isdestreaked = can_run('isdestreaked') or (warn "Can't find isdestreaked" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base);
+my ($outroot, $release, $bytes, $md5sum);
+my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
+
+GetOptions(
+           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,     # camera for evaluating file rules
+           'stage=s'        => \$stage,      # raw, chip, warp, or diff
+           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
+           'path_base=s'    => \$path_base,  # path_base of the input
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database?
+           'no-op'          => \$no_op,      # Don't do any operations?
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base",
+           -exitval => 3) unless
+    defined $magic_ds_id and
+    defined $camera and
+    defined $stage and
+    defined $stage_id and
+    defined $path_base;
+
+$component = "exposure";
+
+if ($camera ne 'GPC1') {
+    my_die("camera must be GPC1 not $camera ",  $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+}
+
+if ($stage ne 'camera') {
+    my_die("stage must be camera",  $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+}
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
+
+my $nebulousServer = $ENV{NEB_SERVER};
+&my_die("cannot find NEB_SERVER in environment", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer;
+
+my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); };
+if ($@ or not defined $nebulous) {
+    &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $dirname  = dirname($path_base);
+my $basename = basename($path_base);
+my $nebulousInput = inNebulous($dirname);
+
+&my_die ("$dirname is not in nebulous", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) unless $nebulousInput;
+
+my $backup_path_base = $dirname   . "/SR_" . "$basename";
+my $recovery_path_base = $dirname . "/REC_" . "$basename";
+
+my $astrom =  $ipprc->filename("PSASTRO.OUTPUT", $path_base);
+my $bastrom = $ipprc->filename("PSASTRO.OUTPUT", $backup_path_base);
+
+print "swap $bastrom and $astrom\n";
+restore_file($astrom, $bastrom, 1) or
+        &my_die("failed to restore smf file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+
+my @otas = qw(XY01 XY02 XY03 XY04 XY05 XY06 XY10 XY11 XY12 XY13 XY14 XY15 XY16 XY17 XY20 XY21 XY22 XY23 XY24 XY25 XY26 XY27 XY30 XY31 XY32 XY33 XY34 XY35 XY36 XY37 XY40 XY41 XY42 XY43 XY44 XY45 XY46 XY47 XY50 XY51 XY52 XY53 XY54 XY55 XY56 XY57 XY60 XY61 XY62 XY63 XY64 XY65 XY66 XY67 XY71 XY72 XY73 XY74 XY75 XY76);
+
+# go fix the mask files
+foreach my $class_id (@otas) {
+    my $mask  = "$path_base.$class_id.mk.fits";
+    my $bmask = "$backup_path_base.$class_id.mk.fits";
+    my $rmask = "$recovery_path_base.$class_id.mk.fits";
+    restore_file($mask, $bmask, 0);
+#  or        &my_die("failed to restore $mask", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+{
+    # this sets the camRun's magicked value back to zero
+    my $command = "$magicdstool -revertdestreakedfile -i_am_sure";
+    $command   .= " -state goto_restored";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -component $component";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, $component,
+                $error_code);
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+}
+exit 0;
+
+### Pau.
+sub restore_file {
+    my $original_url = shift;
+    my $backup_url = shift;
+    my $smf = shift;
+
+    my $original = check_file($original_url, 1);
+
+    my $swap_urls = 0;
+    my $kill_censored = 0;
+    my $replicate = 0;
+
+    if ($original->{so_exists}) {
+        if ($original->{num_instances} == 0 or $original->{censored}) {
+            # expected case: censored original
+            # check the status of the backup
+            my $backup = check_file($backup_url, 1);
+            if ($backup->{so_exists}) {
+                if (!$backup->{censored}) {
+                    # expected case backup is uncensored
+                    if ($backup->{inconsistent}) {
+                        if (!repair_file($backup), $smf) {
+                            print STDERR "failed to repair uncensored backup file $backup_url\n";
+                            return 0;
+                        }
+                    }
+                    # All systems go
+                    $swap_urls = 1;
+                    $kill_censored = 1;
+                    $replicate = $backup->{num_instances} < 2;
+                } else {
+                    # user intervention required
+                    print STDERR "both original and backup url point to censored file names\n";
+                    my $name = $original->{instances}->[0];
+                    print STDERR "    original: $name\n";
+                    $name = $backup->{instances}->[0];
+                    print STDERR "    backup:   $name\n";
+                    return 0;
+                }
+            } else {
+                print STDERR "no backup file exists: $backup_url\n";
+                return 0;
+            }
+        } else {
+            # original exists but does not appear to be censored. Just have a quick look at the backup
+            my $backup = check_file($backup_url, 0);
+            if ($backup->{so_exists} and $backup->{num_instances} and !$backup->{censored}) {
+                # user intervention required
+                print STDERR "both original and backup url point to un-censored file names\n";
+                my $name = $original->{instances}->[0];
+                print STDERR "    original: $name\n";
+                $name = $backup->{instances}->[0];
+                print STDERR "    backup:   $name\n";
+                return 0;
+            }
+            if ($original->{inconsistent} or scalar @{$file->{bad_instances}} > 0) {
+                if (!repair_file($original, $smf)) {
+                    print STDERR "failed to repair uncensored original file $original_url\n";
+                    return 0;
+                }
+            }
+            $kill_censored = 1;
+            $replicate = $original->{num_instances} < 2;
+        }
+    } else {
+        # no original storage object exists
+        # We could check for a backup and rename it but this should be a very unusual case
+        print STDERR "No storage object found for $original_url\n";
+        return 0;
+    }
+
+    if (!$no_op) {
+        if ($swap_urls) {
+            if (! $nebulous->swap($backup_url, $original_url) ) {
+                print "failed to swap $backup_url\n";
+                print "            to $original_url\n";
+                &my_die( "aborting", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+            } else {
+                print "restored $original_url\n";
+            }
+        }
+        if ($kill_censored) {
+            # destroy the backup file
+            $ipprc->kill_file($backup_url);
+        }
+
+        if (!$replicate) {
+            $ipprc->replicate_file($original_url) 
+                or &my_die( "failed to replicate $original_url", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+        }
+    } else {
+        print "Skipping:\n";
+        print "  Swap urls $backup_url $original_url\n" if $swap_urls;
+        print "  Delete $backup_url\n" if $swap_urls;
+        print "  Replicate $original_url\n" if $replicate;
+    }
+
+    return 1;
+}
+
+sub check_file {
+    my $url = shift;
+    my $full_check = shift;
+
+    my %file;
+    $file{url} = $url;
+    $file{so_exists} = $nebulous->storage_object_exists($url);
+    $file{total_instances} = 0;
+    $file{num_instances} = 0;
+    $file{censored} = 0;
+    $file{inconsistent} = 0;
+    $file{instances} = [];
+    $file{bad_instances} = [];
+    $file{size} = 0;
+    $file{md5} = 0;
+
+    if ($file{so_exists}) {
+        my $stat = $nebulous->stat($url);
+
+        &my_die("nebulous stat failed for $url", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) unless $stat;
+
+        $file{total_instances} = $stat->[7];
+
+        if ($file{total_instances}) {
+            my $instances;
+            eval {
+                $instances = $nebulous->find_instances($url, 'any');
+            };
+            if (!$instances or !scalar @$instances) {
+                return \%file;
+            }
+
+            my $first_inst;
+            my $first_size = 0;
+            my $first_md5 = "";
+            my $first_censored ;
+            my $available_instances = 0;
+            foreach my $inst (@$instances) {
+                $inst =~ s/file\:\/\///g;
+
+                if ( ! -e $inst) {
+                    print STDERR "instance file $inst does not exist\n";
+                    push @{$file{bad_instances}}, $inst;
+                    next;
+                }
+                my $this_size = -s $inst;
+                if (!$this_size) {
+                    print STDERR "instance file $inst has zero size\n";
+                    push @{$file{bad_instances}}, $inst;
+                    next;
+                }
+                $available_instances++;
+                push @{$file{instances}}, $inst;
+                my $this_md5 = $full_check ? file_md5_hex($inst) : "";
+                if (!defined $first_inst) {
+                    $first_inst = $inst;
+                    $first_size = $this_size;
+                    $first_md5 = $this_md5;
+                    $first_censored = ($inst =~ /SR_/);
+                    # unless full_check all we really care about is the censored status
+                    last if !$full_check;
+                } elsif ($this_size != $first_size) {
+                    print STDERR "size of $inst $this_size is not equal to size of $first_inst: $first_size\n";
+                    $file{inconsistent} = 1;
+                } elsif ($this_md5 ne $first_md5) {
+                    print STDERR "md5 of $inst $this_md5 is not equal to size of $first_inst: $first_md5\n";
+                    $file{inconsistent} = 1;
+                }
+            }
+            $file{num_instances} = $available_instances;
+            $file{censored} = $first_censored;
+            $file{size} = $first_size;
+            $file{md5} = $first_md5;
+        }
+    }
+
+    return \%file;
+}
+
+sub repair_file {
+    my $file = shift;
+    my $smf = shift;
+    if ($file->{num_instances} < 2) {
+        print STDERR "cannot repair file with only one instance\n";
+        return 0;
+    }
+    my $good_inst;
+    my $good_size;
+    my $good_md5;
+    # try and find a "good instance"
+    my @good;
+    my $i;
+    for ($i = 0; $i < scalar @{$file->{instances}}; $i++; {
+        # assume the worst
+        $good[$i] = 0;
+        if (! -e $inst ) {
+            next;
+        }
+        my $size = -s $inst;
+        if (! $size ) {
+            next;
+        }
+        my $md5 = file_md5_hex($inst);
+        if ($good_inst) {
+            if (($size == $good_size) and ($md5 eq $good_md5)) {
+                $good[$i] = 1;
+            }
+            next;
+        }
+            
+        if (!$smf) {
+            my $command = "$checkfits $inst";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            if ($success) {
+                if (!$good_inst) {
+                    $good_inst = $inst;
+                    $good_$md5 = $md5;
+                    $good_size = $size;
+                }
+                $good[$i] = 1;
+            }
+        } else {
+            # Need to write this program
+            my $command = "$checksmf $inst";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            if ($success) {
+                if (!$good_inst) {
+                    $good_inst = $inst;
+                    $good_$md5 = $md5;
+                    $good_size = $size;
+                }   
+                $good[$i] = 1;
+            }
+        }
+    }
+
+    if (!$good_inst) {
+        print STDERR "unable to find any good instances of $file->{url}\n";
+        return 0;
+    }
+
+    my $num_good = 0;
+    for (my $j = 0; $j <= $i; $j++) {
+        $num_good++ if $good[$j];
+    }
+    for (my $j = 0; $j <= @{$file->{bad_instances}}; $j++) {
+        $good[$i++] = 0;
+    }
+        
+
+    return 0;
+}
+
+
+
+sub delete_recovery_files
+{
+    foreach my $file (@_) {
+        next if !$file;
+	# don't care if this fails or not (it will fail if storage
+	# object doesn't exist
+        $ipprc->kill_file($file);
+    }
+}
+
+sub inNebulous
+{
+    my $path = shift;
+
+    my $scheme = file_scheme($path);
+
+    return $scheme and ($scheme eq "neb");
+}
+
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $magic_ds_id = shift;    # Magic DS identifier
+    my $component = shift;      # class_id or skycell_id
+    my $exit_code = shift;      # Exit code to add
+
+    carp($msg);
+
+    $component = "exposure";
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    # fault the whole run if one of the components fails to revert
+    my $command = "$magicdstool -updaterun";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -set_state failed_restore";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $magic_ds_id");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
+    exit $exit_code;
+}
+
+__END__