Index: trunk/ippScripts/scripts/magic_destreak_cleanup.pl
===================================================================
--- trunk/ippScripts/scripts/magic_destreak_cleanup.pl	(revision 25822)
+++ trunk/ippScripts/scripts/magic_destreak_cleanup.pl	(revision 25822)
@@ -0,0 +1,267 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename dirname );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+use Nebulous::Client;
+use DBI;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $magicdstool   = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($magic_ds_id, $camera);
+my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
+
+GetOptions(
+           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,     # camera for evaluating file rules
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database?
+           'no-op'          => \$no_op,      # Don't do any operations?
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --magic_ds_id --camera",
+           -exitval => 3) unless
+    defined $magic_ds_id and
+    defined $camera;
+#    defined $stage and
+#    defined $stage_id;
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile;
+
+if (0) {
+my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
+&my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer;
+
+my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); };
+if ($@ or not defined $nebulous) {
+    &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR);
+}
+}
+
+$dbname = metadataLookupStr( $ipprc->{_siteConfig}, 'DBNAME' ) if !$dbname;
+&my_die ("Unable to find DBNAME in site or command line arguments", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbname;
+
+my $dbuser = metadataLookupStr( $ipprc->{_siteConfig}, 'DBUSER' );
+&my_die ("Unable to find DBUSER in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbuser;
+
+my $dbpassword = metadataLookupStr( $ipprc->{_siteConfig}, 'DBPASSWORD' );
+&my_die ("Unable to find DBPASSWORD in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbpassword;
+
+my $dbserver = metadataLookupStr( $ipprc->{_siteConfig}, 'DBSERVER' );
+&my_die ("Unable to find DBSERVER in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbserver;
+
+my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to mysql server\n";
+
+my $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, magicRun.inverse"
+         . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id)"
+         . " WHERE magic_ds_id = $magic_ds_id";
+my $q2 = "SELECT * from magicDSFile where magic_ds_id = $magic_ds_id";
+
+# my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $stmt1 = $dbh->prepare($q1);
+$stmt1->execute();
+my $nrows = $stmt1->rows;
+&my_die ("Unable to find magicDSRun $magic_ds_id", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nrows;
+my $run = $stmt1->fetchrow_hashref();
+
+my $state = $run->{state};
+my $stage = $run->{stage};
+my $stage_id = $run->{stage_id};
+my $cam_path_base = $run->{cam_path_base};
+my $inverse = $run->{inverse};
+
+
+&my_die("unexpected run state found: $state", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $state ne "goto_purged";
+&my_die("purge not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw";
+
+
+my $stmt2 = $dbh->prepare($q2);
+$stmt2->execute();
+
+my $num_components = 0;
+while (my $comp = $stmt2->fetchrow_hashref()) {
+        $num_components++;
+
+        my $component = $comp->{component};
+        my $backup_path_base = $comp->{backup_path_base};
+        my $recovery_path_base = $comp->{recovery_path_base};
+        my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
+        my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
+
+        if ($stage eq "chip") {
+            # we use the mask output from the camera stage for input and replace
+            # the output of the chip stage with that mask as well.
+            my $cammask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component);
+
+            if ($backup_path_base) {
+                $bimage  = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $component);
+                $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $component);
+                $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $component);
+                # This is kludgey but correct
+                
+                my $prefix;
+                if ($backup_path_base =~ /\.SR/) {
+                    $prefix = "";
+                } else {
+                    $prefix = "SR_";
+                }
+
+                $bmask   = dirname($backup_path_base) . "/" . $prefix . basename($cammask);
+            }
+            if ($recovery_path_base) {
+                $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component);
+                $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
+                $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component);
+                # This is kludgey but correct
+                $rmask   = dirname($recovery_path_base) . "/" . basename($cammask);
+            }
+        } elsif ($stage eq "camera") {
+            if ($backup_path_base) {
+                $bastrom = $ipprc->filename("PSASTRO.OUTPUT", $backup_path_base);
+            }
+        } elsif ($stage eq "warp") {
+            if ($backup_path_base) {
+                $bimage  = $ipprc->filename("PSWARP.OUTPUT", $backup_path_base);
+                $bmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $backup_path_base);
+                $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base);
+                $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
+            } 
+            if ($recovery_path_base) {
+                $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
+                $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
+                $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
+                $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base);
+            } 
+        } elsif ($stage eq "diff") {
+            my $name = "PPSUB.OUTPUT";
+            if ($backup_path_base) {
+                $bimage  = $ipprc->filename($name, $backup_path_base);
+                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
+                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
+                $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+            }
+            if ($recovery_path_base) {
+                $rimage  = $ipprc->filename($name, $recovery_path_base);
+                $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
+                $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
+                $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
+            }
+        }
+
+        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
+
+        if ($stage eq "diff" and $inverse) {
+            my $name = "PPSUB.INVERSE";
+            if ($backup_path_base) {
+                $bimage  = $ipprc->filename($name, $backup_path_base);
+                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
+                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
+                $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+            }
+            if ($recovery_path_base) {
+                $rimage  = $ipprc->filename($name, $recovery_path_base);
+                $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
+                $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
+                $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
+            }
+            delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources, undef);
+        }
+}
+
+if (!$no_update and ($num_components > 0)) {
+    my $result = $dbh->do("DELETE FROM magicDSFile WHERE magic_ds_id = ?", undef, $magic_ds_id);
+    # my $result = $stmt3->do($magic_ds_id);
+    my_die("attempt to delete magicDSFiles failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0";
+}
+
+if (!$no_update) {
+    my $result = $dbh->do("UPDATE magicDSRun SET state = 'purged' WHERE magic_ds_id = ?", undef, $magic_ds_id);
+    my_die("attempt to update magicDSRun failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0";
+} else {
+    print STDERR "skipping update of magicDSRun\n";
+}
+
+
+### Pau.
+
+sub delete_files {
+    foreach my $file (@_) {
+        if ($file) {
+            if ($ipprc->file_exists($file)) {
+                if (!$no_update) {
+                    print STDERR "deleting $file\n" if $verbose;
+                    $ipprc->file_delete($file) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR);
+                    } else {
+                    print STDERR "skipping delete $file\n";
+                }
+            } else {
+                print STDERR "$file not found\n" if $verbose;
+            }
+        }
+    }
+}
+
+
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $magic_ds_id = shift;    # Magic DS identifier
+    my $exit_code = shift;      # Exit code to add
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    my $command = "$magicdstool -updaterun -set_state purge_fault.$exit_code";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless (1 or $no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $magic_ds_id");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
+    carp($msg);
+    exit $exit_code;
+}
+
+__END__