Changeset 29515 for branches/eam_branches/ipp-20100823/ippScripts

Timestamp:

Oct 21, 2010, 2:45:13 PM (16 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20100823/ippScripts/scripts

Files:

• automate_stacks.pl (modified) (11 diffs)
• ipp_apply_burntool.pl (modified) (2 diffs)
• ipp_cleanup.pl (modified) (8 diffs)
• magic_destreak.pl (modified) (9 diffs)
• magic_process.pl (modified) (15 diffs)
• magic_tree.pl (modified) (1 diff)

branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl

@@ -124 +124 @@
     defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or
     defined $test_mode or defined $clean_old or defined $check_mode or
-    defined $confirm_stacks;
+    defined $confirm_stacks or defined $burntool_stats;
 
 # Configurable parameters from our config file.
 my @target_list = ();
 my @filter_list = ();
+my %distribution_list = ();
 my %tessID_list = ();
 my %obsmode_list = ();
@@ -135 +136 @@
 my %cleanmods_list = ();
 my %stackable_list = ();
+my %extra_processing = ();
 my %reduction_class = ();
 my %macro_formats = ();
@@ -206 +208 @@
                 $this_target = ${ $tentry }{value};
                 push @target_list, $this_target;
+                $distribution_list{$this_target} = $this_target;
+            }
+            elsif (${ $tentry }{name} eq 'DISTRIBUTION') {
+                $distribution_list{$this_target} = ${ $tentry }{value};
             }
             elsif (${ $tentry }{name} eq 'TESS') {
@@ -222 +228 @@
                 $stackable_list{$this_target} = ${ $tentry }{value};
             }
+            elsif (${ $tentry }{name} eq 'EXTRA_PROCESSING') {
+                $extra_processing{$this_target} = ${ $tentry }{value};
+            }
             elsif (${ $tentry }{name} eq 'REDUCTION') {
                 $reduction_class{$this_target} = ${ $tentry }{value};
@@ -726 +735 @@
         my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target);
         if (defined($burntool_stats)) {
-            print "$Nexposures $Nimfiles $Nburntooled $Nalready\n";
+            print "BTSTATS: $date $target $Nexposures $Nimfiles $Nburntooled $Nalready\n";
         }
 
         if ($Nexposures == 0) {
             print STDERR "execute_chips: Target $target on $date had no exposures.\n";
-#           next;
+            next;
         }
         if ($Nalready != 0) {
             print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n";
-#            next;
+            next;
         }
         if ($Nimfiles != $Nburntooled) {
@@ -1183 +1192 @@
         &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     }
+    
+    
     return(0);
 }
@@ -1190 +1201 @@
     my $pretend = shift;
     foreach my $target (@target_list) {
-        if ($stackable_list{$target} == 1) {
-            foreach my $filter (@filter_list) {
+        foreach my $filter (@filter_list) {
+            if (exists($extra_processing{$target})) {
+                do_extra_processing($date,$target,$filter,$pretend);
+            }
+            if ($stackable_list{$target} == 1) {
                 my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
                 if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
@@ -1231 +1245 @@
                 }
             }
+            else {
+                # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
+            }
         }
-        else {
-            # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
-        }
-    }
-
+    }
 }
 
@@ -1291 +1304 @@
 
 #
+# Extra processing
+################################################################################
+
+sub do_extra_processing {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    if ($target eq 'OSS') {
+        my $db = init_gpc_db();
+
+        my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+        $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+        print STDERR "$obj_sth\n";
+        my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+        foreach my $object_row (@{ $object_ref }) {
+            my $this_object = shift @{ $object_row };
+            my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+            $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+            $input_sth .= " ORDER BY dateobs ";
+
+            my $warps = $db->selectall_arrayref( $input_sth );
+            
+            if (($#{ $warps } + 1) % 2 != 0) {
+                print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n";
+                die;
+            }
+            
+            while ($#{ $warps } > -1) {
+                my $input_warp = shift @{ $warps };
+                my $template_warp = shift @{ $warps };
+                my $input_exp_id = ${ $input_warp }[0];
+                my $template_exp_id = ${ $template_warp }[0];
+                
+                my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
+                $cmd .= "-input_label $label  -template_label $label ";
+                $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+                $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
+                $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+#               $cmd .= " -pretend ";
+                if (defined($pretend)) {
+                    $cmd .= ' -pretend ';
+                }
+                if ($debug == 1) {
+                    $cmd .= ' -pretend ';
+                }
+                print STDERR "EXTRA_PROCESSING: $cmd\n";
+                if (($debug == 0)&&(!defined($pretend))) {
+                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run ( command => $cmd, verbose => $verbose );
+                    unless ($success) {
+                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                        &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                    }
+                }
+            }
+        }
+    }
+}
+
+            
+
+
+#
 # Auto-Clean
 ################################################################################
@@ -1380 +1460 @@
     my $object   = $object_list{$target};
     my $comment = $comment_list{$target};
-    my $dist_group = $target;
+    my $dist_group = $distribution_list{$target};
     my $data_group = "${target}.${trunc_date}";
     my $tess_id = $tessID_list{$target};
@@ -1427 +1507 @@
         my $N = $metadata_out{N_MACROS};
         $metadata_out{"ns${N}Macro"} = $macro_formats{$proc_mode};
-        print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
+#       print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
         if (defined($date)&&(defined($target))) {
 

branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_apply_burntool.pl

@@ -55 +55 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1);
+my $nebreplicate = can_run('neb-replicate') or (warn "Can't find neb-replicate" and $missing_tools = 1);
 #my $fpack    = can_run('fpack')    or (warn "Can't find fpack" and $missing_tools = 1);
 if ($missing_tools) {
@@ -214 +215 @@
 
         $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
+        $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
         $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN);
         if ($status) {

branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_cleanup.pl

@@ -53 +53 @@
 }
 
+# set this to 1 to enable checking for files on dead nodes
+# it is off for now because the implementation is a hack
+# See comments below.
+my $check_for_gone = 0;
+
 my $error_state;
-if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned";  }
-if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; }
-if ($mode eq "goto_purged")   { $error_state = "error_purged";   }
+my $done_state;
+if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned"; $done_state = "cleaned"; }
+if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; $done_state = "scrubbed";}
+if ($mode eq "goto_purged")   { $error_state = "error_purged";   $done_state = "purged";}
 
 
@@ -92 +98 @@
     }
 
-    # if there are no chipProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
-    # XXX Why? This could just mean there's nothing to cleanup, or that we're trying to rerun an errored run.
+    # if there are no chipProcessedImfiles (@$stdout_buf == 0) then assume that we're done
+    # it could be that there are no chipProcessedImfiles at all if say if a run was changed from drop to goto_cleaned
+    # or of a run was set to update and then back to goto_cleaned before any images were processed
     if (@$stdout_buf == 0)  {
-        my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state";
+        my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $done_state";
         $command .= " -dbname $dbname" if defined $dbname;
 
@@ -126 +133 @@
 
                 unless ($ipprc->file_exists($config_file)) {
-                    print STDERR "skipping cleanup for chipRun $stage_id $class_id "
-                        . " because config file ($config_file) is missing\n";
-                    $status = 0;
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup for chipRun $stage_id $class_id "
+                            . " because config file ($config_file) is gone\n";
+                    } else {
+                        print STDERR "skipping cleanup for chipRun $stage_id $class_id "
+                            . " because config file ($config_file) is missing\n";
+                        $status = 0;
+                    }
                 }
             }
@@ -354 +366 @@
 
     if (@$stdout_buf == 0) {
-        # No skycells were found for some reason.
-        # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
-        my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state";
+        # No skycells were found for some reason. 
+        # it could be that there are no warpSkyfiles at all if say if a run was changed from drop to goto_cleaned
+        # or of a run was cleaned, set to update, and then back to goto_cleaned before any images were successfully
+        # updated
+        my $command = "$warptool -updaterun -warp_id $stage_id -set_state $done_state";
         $command .= " -dbname $dbname" if defined $dbname;
 
@@ -384 +398 @@
 
                 unless ($ipprc->file_exists($config_file)) {
-                    print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
-                        " because config file is missing\n";
-                    $status = 0;
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" .
+                            " because config file is gone\n";
+                    } else {
+                        print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
+                            " because config file is missing\n";
+                        $status = 0;
+                    }
                 }
             }
@@ -624 +643 @@
 
     if (@$stdout_buf == 0) {
-        # No skycells were found for some reason.
-        # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
-        my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
+        # No skycells were found for some reason. 
+        # it could be that there are no warpSkyfiles at all if say if a run was changed from drop to goto_cleaned
+        # or of a run was cleaned, set to update, and then back to goto_cleaned before any images were successfully
+        my $command = "$difftool -updaterun -diff_id $stage_id -set_state $done_state";
         $command .= " -dbname $dbname" if defined $dbname;
 
@@ -658 +678 @@
 
                 unless ($ipprc->file_exists($config_file)) {
-                    print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
-                        " because config file ($config_file) is missing\n";
-                    $status = 0;
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup for diffRun $stage_id $skycell_id" .
+                            " because config file ($config_file) is gone\n";
+                    } else {
+                        print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
+                            " because config file ($config_file) is missing\n";
+                        $status = 0;
+                    }
                 }
             }
@@ -1665 +1690 @@
 }
 
+my $whichnode;
+sub file_gone
+{
+    # if $check_for_gone check whether the only instance of file is on a lost volumen
+    # XXX: we don't have a proper interface for this. 
+    # For now try to use Bill's hack the script 'whichnode'
+    return 0 if !$check_for_gone;
+
+    my $file = shift;
+
+    if (!$whichnode) {
+        $whichnode = can_run('whichnode') or
+            &my_die("Can't find whichnode", "chip", $stage_id, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my $command = "$whichnode $file";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform whichnode: $error_code", "chip", $stage_id, $error_code);
+    }
+
+    my @lines = split "\n", (join "", @$stdout_buf);
+    my $numGone = 0;
+    my $numNotGone = 0;
+    foreach my $line (@lines) {
+        chomp $line;
+
+        # output lines are either
+        #   "volume available"
+        # or 
+        #   "volume not available"
+
+        my ($volume, $answer, undef) = split " ", $line;
+        # our hack is if the volume has an X in the name it's gone
+        if ($volume =~ /X/) {
+            print STDERR "$file is on $volume which is gone\n";
+            $numGone++;
+        } elsif ($answer eq 'not') {
+            print STDERR "$file is on $volume which is not available\n";
+            $numNotGone++;
+        } else {
+            print STDERR "unexpected output from whichnode: $line\n";
+        }
+    }
+    # if there are any instances that are not on a gone node return 0
+    if ($numNotGone == 0 and $numGone > 0) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
 sub addFilename
 {

branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak.pl

@@ -42 +42 @@
 # Parse the command-line arguments
 my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction);
+my ($streaks_path_base, $inv_streaks_path_base);
 my ($outroot, $recoveryroot, $magicked);
 my ($replace, $release);
@@ -49 +50 @@
            'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
            'camera=s'       => \$camera,     # camera for evaluating file rules
+           'streaks_path_base=s'      => \$streaks_path_base,    # path_base for streaks data
+           'inv_streaks_path_base=s'  => \$inv_streaks_path_base, #path_base for streaks from inverse diff
+           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
            'streaks=s'      => \$streaks,    # file containing the list of streaks
            'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
@@ -78 +82 @@
     defined $camera and
     defined $streaks and
+    defined $streaks_path_base and
     defined $stage and
     defined $stage_id and
@@ -108 +113 @@
     &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
 }
+$inv_streaks_path_base = undef if defined($inv_streaks_path_base) and ($inv_streaks_path_base eq "NULL");
 $inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
 
@@ -240 +246 @@
     my ($allstreaks_fh, $allstreaks_name);
 
+    # Set name of streaks files from their path_base. Note. ne 'NULL' test is for backward compatability
+    # with runs that don't have path_base set yet
+    if ($streaks_path_base ne 'NULL') {
+        $streaks = "$streaks_path_base.streaks";
+    }
+    if ($inv_streaks_path_base and ($inv_streaks_path_base ne 'NULL')) {
+        $inv_streaks = "$inv_streaks_path_base.streaks";
+    }
+
+    my $streaks_resolved = $ipprc->file_resolve($streaks) or &my_die("failed to resolve streaks file $streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
+    my $inv_streaks_resolved;
+    if ($inv_streaks) {
+        $inv_streaks_resolved = $ipprc->file_resolve($inv_streaks) or &my_die("failed to resolve inverse streaks file $inv_streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
+    }
+        
+
     if ($stage eq "raw") {
         $image = $uri;
@@ -301 +323 @@
         $sources    = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
 
-        if ($inv_streaks) {
+        if ($inv_streaks_resolved) {
             # create a temporary file containing the contents of the
             # two streaks files
@@ -307 +329 @@
                     UNLINK => !$save_temps);
 
-            combine_streaks($allstreaks_fh, $streaks, $inv_streaks);
+            combine_streaks($allstreaks_fh, $streaks_resolved, $inv_streaks_resolved);
 
             # apply the combined streaks to both the forward and inverse diffs
-            $streaks = $allstreaks_name;
+            $streaks_resolved = $allstreaks_name;
         }
     }
 
     {
-        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
 
         $command .= " -stats $statsFile";
@@ -340 +362 @@
         }
     }
-    if (($stage eq "diff") and $inv_streaks) {
+    if (($stage eq "diff") and $inv_streaks_resolved) {
         $image   = $ipprc->filename("PPSUB.INVERSE", $path_base);
         $mask    = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base);
@@ -349 +371 @@
         my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
 
-        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
         $command .= " -stats $invStatsFile";
 

branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_process.pl

@@ -17 +17 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
+use File::Copy;
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -41 +42 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile, $final_outroot);
 
 GetOptions(
@@ -49 +50 @@
            'dbname=s'        => \$dbname,     # Database name
            'baseroot=s'      => \$baseroot,   # Output root name
+           'final-outroot=s' => \$final_outroot,   # location for final outputs
            'save-temps'      => \$save_temps, # Save temporary files?
            'verbose'         => \$verbose,    # Print stuff?
@@ -86 +88 @@
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# list of VerifyStreaks input and output files to copy to nebulous 
+my @verify_outputs = qw(
+clusterPos.txt
+duplicate.png
+mask.png
+original.png
+original.fits
+residual.png
+residual.fits
+clusters.list
+);
 
 ### Get a list of inputs
@@ -311 +325 @@
 }
 
-
 ### Input result into database
 {
@@ -334 +347 @@
 
 if ($node eq "root") {
+    # XXXX: Since we just added the result above, all of these my_dies are going to fail
+    # with a duplicate row error.
+    # see more comments below
     my $streaks_file = "$outroot.streaks";
     my $resolved = $ipprc->file_resolve($streaks_file);
@@ -349 +365 @@
     }
 
-    &run_verifystreaks($baseroot);
 
     my $exp_id;                 # Exposure identifier
@@ -366 +381 @@
     }
 
+    &run_verifystreaks($baseroot, $exp_id);
+
     {
         my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file
@@ -383 +400 @@
     }
 
+    my $output_streaks = $final_outroot . ".streaks";
+    if ($output_streaks and ($output_streaks ne $streaks_file)) {
+        copy_to_nebulous($ipprc, $streaks_file, $output_streaks, 1);
+        foreach my $f (@verify_outputs) {
+            my $src = "$baseroot.verify/${exp_id}_$f";
+            my $dest = "$final_outroot.${f}";
+            copy_to_nebulous($ipprc, $src, $dest, 1);
+        }
+    } else {
+        $output_streaks = $streaks_file;
+    }
 
     {
         my $command = "$magictool -addmask";
         $command   .= " -magic_id $magic_id";
-        $command   .= " -uri $streaks_file";
+        $command   .= " -path_base $final_outroot";
         $command   .= " -streaks $num_streaks";
         $command   .= " -dbname $dbname" if defined $dbname;
@@ -397 +425 @@
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                # This isn't going to work because our result was already entered.
+                # XXX: This my_die isn't going to work because the result for the root node was already
+                # added to the database up above.
+                # There is a magicMask has a fault column. Maybe we should 
+                # use that and add a new revert mode that deletes the magicMask and the magicNodeResult
+                # for the root node.
                 &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
             }
@@ -406 +438 @@
 }
 
+
 ### Pau.
 
@@ -411 +444 @@
 
     my $baseroot = shift;
+    my $exp_id = shift;
 
     unless ($VerifyStreaks) {
@@ -431 +465 @@
     my @files = <$baseroot.*.clusters>;
 
-    unless (open ($FILE, ">$outdir/clusters.list")) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+    my $clusters_list = "$outdir/${exp_id}_clusters.list";
+    unless (open ($FILE, ">$clusters_list")) {
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
@@ -441 +476 @@
     close ($FILE);
     if ($status) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
 
-    my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap";
+    my $command = "$VerifyStreaks --out $outdir --clusters $clusters_list $baseroot.root.streakMap";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -513 +548 @@
 }
 
+# Copy a file to nebulous and optionally replicate it
+# We should consider making this an ipprc function. For now try it here so we can print
+# the right error messages
+sub copy_to_nebulous {
+    my $ipprc = shift;
+    my $src = shift;
+    my $dest = shift;
+    my $replicate = shift;
+
+    print "copying $src to $dest\n";
+
+    $ipprc->file_exists($src) or
+        &my_die("expected output file does not exist: $src", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    # copy the file to it's final destination - which is presumably in nebulous
+    # we delete it so that all instances are deleted in case it has been replicated
+    if ($ipprc->file_exists($dest)) {
+        $ipprc->file_delete($dest) or 
+                &my_die("failed to delete existing file: $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $dest_resolved = $ipprc->file_resolve($dest, 'create');
+    &my_die("failed to resolve $dest", $PS_EXIT_UNKNOWN_ERROR) if !$dest_resolved;
+
+    copy ($src, $dest_resolved) or 
+        &my_die("failed to copy $src to $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    if ($replicate and (file_scheme($dest) eq 'neb')) {
+        my $neb = $ipprc->nebulous();
+        $neb->setxattr($dest, 'user.copies', 2, 'create') or 
+            &my_die("failed to set user.copies for $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+        $neb->replicate($dest) or
+            &my_die("failed to replicate $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+
 sub my_die
 {

branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_tree.pl

@@ -83 +83 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform magictool -inputfile: $error_code", $magic_id, $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to perform magictool -inputskyfile: $error_code", $magic_id, $error_code);
+    }
+
+    my $magictool_output = join "", @$stdout_buf;
+    # if there is no output from magictool that means that there are no
+    # diffSkyfiles with non-zero quality. Set fault to a special value so
+    # that these magicRuns can be recognized and dropped.
+    &my_die("magictool -inputskyfile returned no output. Inputs are probably all bad quality.", $magic_id, 42)
+        if !$magictool_output;
+    my $metadata = $mdcParser->parse($magictool_output) or
         &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR);