Index: trunk/ippScripts/scripts/magic_destreak.pl
===================================================================
--- trunk/ippScripts/scripts/magic_destreak.pl	(revision 20732)
+++ trunk/ippScripts/scripts/magic_destreak.pl	(revision 20761)
@@ -16,4 +16,5 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
+use File::Basename qw( basename );
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -27,19 +28,23 @@
 
 # Parse the command-line arguments
-my ($magic_id, $camera, $streaks, $stage, $class_id, $skycell_id, $tmpdir, $recoverydir);
+my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
+my ($outroot, $recoveryroot);
 my ($replace, $remove);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
 GetOptions(
-           'magic_id=s'     => \$magic_id,   # Magic identifier
-           'camera=s'       => \$camera,
+           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,     # camera for evaluating file rules
            'streaks=s'      => \$streaks,    # file containing the list of streaks
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
-           'class_id=s'     => \$class_id,
-           'skycell_id=s'   => \$skycell_id,
-           'tmpdir=s'       => \$tmpdir,     # "directory" for temporary images (may be nebulous)
-           'recoverydir=s'  => \$recoverydir,# "directory" for saving the images of excised pixels
-           'replace'        => \$replace,    # replace the input images with the results.
-           'remove'         => \$remove,     # remove the original images when done YIKES!
+           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
+           'component=s'    => \$component,  # the class_id or skycell_id
+           'uri=s'          => \$uri,        # uri of the input image
+           'path_base=s'    => \$path_base,  # path_base of the input
+           'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
+           'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
+           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'replace=s'        => \$replace,    # replace the input images with the results.
+           'remove=s'         => \$remove,     # remove the original images when done YIKES!
            'save-temps'     => \$save_temps, # Save temporary files?
            'dbname=s'       => \$dbname,     # Database name
@@ -51,18 +56,27 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --magic_id --camera --streaks --stage --tmpdir",
+pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --outroot",
            -exitval => 3) unless
-    defined $magic_id and
+    defined $magic_ds_id and
     defined $camera and
     defined $streaks and
     defined $stage and
-    defined $tmpdir;
+    defined $stage_id and
+    defined $component and
+    defined $outroot;
     
-if (!defined($class_id) and (($stage eq "raw") or ($stage eq "chip"))) {
-    carp ("--class_id is required for stage $stage");
-    exit  $PS_EXIT_PROG_ERROR;
-} elsif (!defined($skycell_id) and (($stage eq "warp") or ($stage eq "diff"))) {
-    carp ("--skycell_id is required for stage $stage");
-    exit  $PS_EXIT_PROG_ERROR;
+
+my ($skycell_args, $class_id, $skycell_id);
+
+if (($stage eq "raw") or ($stage eq "chip")) {
+    $class_id = $component;
+    $skycell_args = " -class_id $component";
+} elsif ($stage eq "warp") {
+    $skycell_id = $component;
+    $skycell_args = " -skycell_id $component";
+} elsif ($stage eq "diff") {
+    $skycell_id = $component;
+} else {
+    &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
 }
 
@@ -74,6 +88,8 @@
 # Look for programs we need
 my $missing_tools;
-my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $magicdstool   = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
 my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1);
+# todo get rid of this dependence on magic_id
+my $magictool   = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -83,31 +99,40 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $magictool_args;
-my $skycell_args;
-my $component;
-if ($stage eq "raw") {
-    $magictool_args = "-rawimfile -class_id $class_id";
-    $skycell_args = " -class_id $class_id";
-    $component = $class_id;
-} elsif ($stage eq "chip") {
-    $magictool_args = "-chipprocessedimfile -class_id $class_id";
-    $skycell_args = " -class_id $class_id";
-    $component = $class_id;
-} elsif ($stage eq "warp") {
-    $magictool_args = "-warpskyfile -skycell_id $skycell_id";
-    $skycell_args = " -skycell_id $skycell_id";
-    $component = $skycell_id;
-} elsif ($stage eq "diff") {
-    $magictool_args = "-diffskyfile -skycell_id $skycell_id";
-    $component = $skycell_id;
-} else {
-    carp("invalid stage value found: $stage\n");
-    exit  $PS_EXIT_PROG_ERROR;
-}
-
-### Get the input information
-my $input;          
-{
-    my $command = "$magictool -magic_id $magic_id $magictool_args";
+my $basename = basename($path_base);
+
+#
+# parse the replace and remove arguments check for errors and set upt
+# the appropriate paths
+if (defined($replace) and ($replace eq "F")) {
+    $replace = 0;
+}
+
+if (($stage eq "raw") and $replace and ! $recoveryroot) {
+    &my_die("Can not replace raw files without defining recoveryroot.",
+        $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+if (defined($remove) and ($remove eq "F")) {
+    $remove = 0;
+}
+
+if ($remove and !$replace) {
+    &my_die("cannot remove without replace", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+my $backup_uri;
+if (! $remove) {
+    $backup_uri = "$outroot/$basename";
+}
+
+my $recovery_uri;
+if ($recoveryroot) {
+    $recovery_uri = "$recoveryroot/$basename";
+}
+
+# get skycell list if needed
+my ($sfh, $skycell_list);          
+if ($skycell_args) {
+    my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args";
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -115,37 +140,15 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform magictool $magictool_args: $error_code", $magic_id, $component, $error_code);
+        &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
     }
 
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $magic_id, $component, $PS_EXIT_PROG_ERROR);
-
-    my $input_list = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $magic_id, $component, $PS_EXIT_PROG_ERROR);
-
-    $input = $input_list->[0];
-}
-
-# get skycell list if needed
-my ($sfh, $skycell_list);          
-if ($skycell_args) {
-    my $command = "$magictool -magic_id $magic_id -diffskyfile $skycell_args";
-    $command .= " -class_id $class_id" if $class_id;
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform magictool -diffskyfile $skycell_args: $error_code", $magic_id, $component, $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $magic_id, $component, $PS_EXIT_PROG_ERROR);
-
-    my $input_list = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $magic_id, $component, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+    my $skycells = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
     ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
 
-    foreach my $skycell (@$input_list) {
+    foreach my $skycell (@$skycells) {
         print $sfh "$skycell->{uri}\n"
     }
@@ -153,15 +156,14 @@
 }
 
-my $image = $input->{uri};
-my $path_base = $input->{path_base};
+my $image = $uri;
 
 my ($mask, $weight, $astrom);
 
 if ($stage eq "raw") {
-    $astrom = $ipprc->filename("PSASTRO.OUTPUT", $input->{cam_path_base});
+    $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
 } elsif ($stage eq "chip") {
     $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id);
     $weight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $path_base, $class_id);
-    $astrom = $ipprc->filename("PSASTRO.OUTPUT", $input->{cam_path_base});
+    $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
 } elsif ($stage eq "warp") {
     $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
@@ -173,9 +175,9 @@
 
 {
-    my $command = "$streaksremove -stage $stage -tmproot $tmpdir -streaks $streaks -image $image";
+    my $command = "$streaksremove -stage $stage -tmproot $outroot -streaks $streaks -image $image";
     $command .= " -class_id $class_id" if defined $class_id;
-    $command .= " -recovery $recoverydir" if defined $recoverydir;
+    $command .= " -recovery $recoveryroot" if defined $recoveryroot;
     $command .= " -astrom $astrom" if defined $astrom;
-    # skip mask and weight images for now
+    # XXX: remove this   skip mask and weight images for now
 #    $command .= " -mask $mask" if defined $mask;
 #    $command .= " -weight $weight" if defined $weight;
@@ -187,5 +189,5 @@
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform streaksremove: $error_code", $magic_id, $component, $error_code);
+            &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
         }
     } else {
@@ -194,24 +196,28 @@
 }
 
-### Input result into database
-#{
-#    my $command = "$magictool -addresult";
-#    $command   .= " -magic_id $magic_id";
-#    $command   .= " -node $node";
-#    $command   .= " -uri $outroot";
-#    $command   .= " -dbname $dbname" if defined $dbname;
-#
-#    # Add the processed file to the database
-#    unless ($no_update) {
-#        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-#            run(command => $command, verbose => $verbose);
-#        unless ($success) {
-#            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-#            &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code);
-#        }
-#    } else {
-#        print "Skipping command: $command\n";
-#    }
-#}
+# if recovery and/or backup files were expected make sure they exist
+
+# Input result into database
+{
+    my $command = "$magicdstool -adddestreakedfile";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -component $component";
+    $command   .= " -backup_uri $backup_uri" if $backup_uri;
+    $command   .= " -recovery_uri $recovery_uri" if $recovery_uri;
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, $component,
+                $error_code);
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+}
 
 
@@ -223,5 +229,5 @@
 {
     my $file = shift;           # Name of file
-    &my_die("Unable to find output file: $file", $magic_id, $component, $PS_EXIT_SYS_ERROR) unless
+    &my_die("Unable to find output file: $file", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless
         $ipprc->file_exists($file);
 }
@@ -230,8 +236,25 @@
 {
     my $msg = shift;            # Warning message on die
-    my $magic_id = shift;       # Magic identifier
+    my $magic_ds_id = shift;    # Magic DS identifier
     my $component = shift;      # class_id or skycell_id
     my $exit_code = shift;      # Exit code to add
 
+    my $command = "$magicdstool -adddestreakedfile";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -component $component";
+    $command   .= " -code $exit_code";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $magic_ds_id $component");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
     carp($msg);
     exit $exit_code;