Changeset 27839 for branches/simmosaic_branches/ippScripts/scripts/magic_destreak.pl

Timestamp:

May 3, 2010, 8:45:22 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simmosaic_branches

Files:

• . (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (23 diffs)

branches/simmosaic_branches/ippScripts/scripts/magic_destreak.pl

@@ -8 +8 @@
 use Sys::Hostname;
 my $host = hostname();
+my $date = `date`;
 print "\n\n";
-print "Starting script $0 on $host\n\n";
+print "Starting script $0 on $host at $date\n\n";
 
 use vars qw( $VERSION );
@@ -15 +16 @@
 
 use IPC::Cmd 0.36 qw( can_run run );
-use File::Temp qw( tempfile );
+use File::Temp qw( tempfile tempdir );
 use File::Basename qw( basename dirname );
 use PS::IPP::Metadata::Config;
@@ -32 +33 @@
 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
 my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -38 +41 @@
 
 # Parse the command-line arguments
-my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base);
+my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction);
 my ($outroot, $recoveryroot);
 my ($replace, $release);
@@ -47 +50 @@
            'camera=s'       => \$camera,     # camera for evaluating file rules
            'streaks=s'      => \$streaks,    # file containing the list of streaks
+           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
+           'exp_id=s'       => \$exp_id,     # exp_id, chip_id, warp_id, or diff_id
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
@@ -52 +57 @@
            'uri=s'          => \$uri,        # uri of the input image
            'path_base=s'    => \$path_base,  # path_base of the input
-           'inverse'        => \$inverse,    # Inverse subtraction?
            'cam_path_base=s'=> \$cam_path_base,  # path_base from camera stage (for chip and raw)
+           'cam_reduction=s'=> \$cam_reduction,  # reduction class from camera stage (for chip and raw)
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
            'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
@@ -74 +79 @@
     defined $stage and
     defined $stage_id and
+    defined $exp_id and
     defined $component and
     defined $uri and
@@ -81 +87 @@
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
+
+$cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL');
+
+my $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use
+&my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
 
 my ($skycell_args, $class_id, $skycell_id);
@@ -95 +106 @@
     &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
 }
+$inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -110 +122 @@
     if ($stage eq 'camera') {
         my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
-        &my_die("cannot find NEB_SERVER in site configuration", 
-                                        $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) 
+        &my_die("cannot find NEB_SERVER in site configuration",
+                                        $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR)
             if !$ nebulousServer;
 
@@ -144 +156 @@
     }
 }
+
+my $statsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.stats";
 
 my $backup_path_base;
@@ -165 +179 @@
         if (! -e $recoveryroot ) {
             my $code = system "mkdir -p $recoveryroot";
-            &my_die("cannot create output directory $outroot", $magic_ds_id, $component,
+            &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
                     $code >> 8) if $code;
         }
@@ -172 +186 @@
     $recovery_path_base = "$recoveryroot/$basename/";
 }
+
+my $temp_dir;
 
 if ($stage ne "camera") {
@@ -194 +210 @@
                     &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
 
+            $temp_dir = tempdir( CLEANUP => !$save_temps);
             ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
 
             foreach my $skycell (@$skycells) {
-                my $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                my $skycell_uri;
+                if ($skycell->{data_state} eq "full") {
+                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                } else {
+                    # diff run must have been cleaned up, need to create this skycell file on the fly
+                    my $skycell_id = $skycell->{skycell_id};
+                    $skycell_uri = "$temp_dir/$skycell_id";
+                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
+                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+                }
                 print $sfh "$skycell_uri\n";
             }
-            close $sfh;
+            close $sfh
         }
     }
 
     my ($image, $mask, $ch_mask, $weight, $astrom, $sources);
+
+    # if we're destreaking a bothways diff need to combine the
+    # two streaks files
+    my ($allstreaks_fh, $allstreaks_name);
 
     if ($stage eq "raw") {
@@ -211 +241 @@
         $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id) if $release ;
     } elsif ($stage eq "chip") {
+
+        # Check to see if we're using dynamic masks
+        my $dynamicMasks;               # Use dynamic masks?
+        {
+            # Get the PSASTRO recipe
+            my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform ppConfigDump: $error_code", $magic_ds_id, $component,
+                        $PS_EXIT_CONFIG_ERROR);
+            }
+            my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+                &my_die("Unable to parse metadata config doc", $magic_ds_id, $component,
+                        $PS_EXIT_CONFIG_ERROR);
+
+            $dynamicMasks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
+        }
+
         # we use the mask output from the camera stage for input and replace
         # the output of the chip stage with that mask as well.
         $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
-        $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
-        $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
         $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
         $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
-        if (!$ipprc->file_exists($mask)) {
-            carp("camera mask file $mask for $component not found. Continuing using mask from chip stage.");
-            $mask = $ch_mask;
-            $ch_mask = undef;
-        }
+
+        if ($dynamicMasks) {
+            $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
+            $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
+        } else {
+            $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
+        }
+
         # XXX: should we censor the sources as well? For now we're leaving them out of the distribution bundles
         # because they confuse people
@@ -234 +285 @@
         $sources    = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $path_base);
     } elsif ($stage eq "diff") {
-        my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images
-        $image  = $ipprc->filename($name, $path_base);
-        $mask   = $ipprc->filename("$name.MASK", $path_base);
-        $weight = $ipprc->filename("$name.VARIANCE", $path_base);
+        $image  = $ipprc->filename("PPSUB.OUTPUT", $path_base);
+        $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
+        $weight = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
         $sources    = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
+
+        if ($inv_streaks) {
+            # create a temporary file containing the contents of the
+            # two streaks files
+            ($allstreaks_fh, $allstreaks_name) = tempfile ("/tmp/all.streaks.XXXX",
+                    UNLINK => !$save_temps);
+
+            combine_streaks($allstreaks_fh, $streaks, $inv_streaks);
+
+            # apply the combined streaks to both the forward and inverse diffs
+            $streaks = $allstreaks_name;
+        }
     }
 
@@ -244 +306 @@
         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
 
+        $command .= " -stats $statsFile";
         $command .= " -class_id $class_id" if defined $class_id;
         $command .= " -recovery $recoveryroot" if defined $recoveryroot;
@@ -266 +329 @@
         }
     }
+    if (($stage eq "diff") and $inv_streaks) {
+        $image   = $ipprc->filename("PPSUB.INVERSE", $path_base);
+        $mask    = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base);
+        $weight  = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $path_base);
+        $sources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $path_base);
+
+        # Note: we create a stats file for the inverse procesing but we don't look at the results
+        my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
+
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        $command .= " -stats $invStatsFile";
+
+        $command .= " -recovery $recoveryroot" if defined $recoveryroot;
+        $command .= " -mask $mask" if defined $mask;
+        $command .= " -weight $weight" if defined $weight;
+        $command .= " -sources $sources" if defined $sources;
+        $command .= " -replace" if $replace;
+        $command .= " -release" if $release;
+        $command .= " -dbname $dbname" if defined $dbname;
+        unless (defined $no_op) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
+            }
+        } else {
+            print "skipping command $command\n";
+        }
+    }
 } else {
     # camera stage. The only work to do is to censor the detections file
@@ -293 +386 @@
         my $quality = $file->{quality};
         if (!$quality) {
-            my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id); 
+            my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id);
             if (! $ipprc->file_exists($mask)) {
                 # camera mask doesn't exist for this chip. Fall back to the chip mask
@@ -315 +408 @@
                 &my_die("Unable to perform censorObjects: $error_code", $magic_ds_id, $component, $error_code);
             }
-            my $output = $ipprc->filename("CENSOR.OUTPUT", $backup_path_base); 
+            my $output = $ipprc->filename("CENSOR.OUTPUT", $backup_path_base);
             &my_die("expected output file $output not found ", $magic_ds_id, $component, $PS_EXIT_DATA_ERROR)
                 unless $ipprc->file_exists($output);
@@ -329 +422 @@
 }
 
-# if recovery and/or backup files were expected make sure they exist
+my $statsFlags;
+if ($stage ne "camera") {
+    file_check($statsFile);
+    {
+        my $resolvedStatsFile = $ipprc->file_resolve($statsFile);
+        my $command = "$ppStatsFromMetadata $resolvedStatsFile - STREAKSREMOVE";
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform ppStatsFromMetadata: $error_code", $magic_ds_id, $component, $error_code);
+        }
+        foreach my $line (@$stdout_buf) {
+            $statsFlags .= " $line";
+        }
+        chomp $statsFlags;
+    }
+}
+
+# XXX: if recovery and/or backup files were expected make sure they exist
 
 # Input result into database
@@ -339 +452 @@
     $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
     $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
+    $command   .= " $statsFlags" if $statsFlags;
     $command   .= " -dbname $dbname" if defined $dbname;
 
@@ -376 +490 @@
 }
 
+sub combine_streaks
+{
+    my $fout = shift;
+    my $fn1 = shift;
+    my $fn2 = shift;
+
+    my ($n1, @streaks1) = read_streaks_file($fn1);
+    my_die("failed to read streaks from $fn1", $magic_ds_id, $component,
+        $PS_EXIT_UNKNOWN_ERROR) if $n1 < 0;
+
+    my ($n2, @streaks2) = read_streaks_file($fn2);
+    my_die("failed to read streaks from $fn2", $magic_ds_id, $component,
+        $PS_EXIT_UNKNOWN_ERROR) if $n2 < 0;
+
+    print $fout $n1 + $n2 . "\n";
+
+    foreach my $line (@streaks1, @streaks2) {
+        print $fout $line;
+    }
+
+    close $fout
+        or my_die("failed to close combined streaks file", $magic_ds_id,
+                     $component, $PS_EXIT_UNKNOWN_ERROR);
+}
+
+sub read_streaks_file
+{
+    my $filename = shift;
+    my $fh;
+    open $fh, "<$filename" or my_die("failed to open $filename",
+                    $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+
+    # first line is the number of streaks
+    my $line = <$fh>;
+    chomp $line;
+    my $nstreaks = $line;
+
+    my @streaks;
+
+    foreach $line (<$fh>) {
+        push @streaks, $line;
+    }
+
+    close $fh;
+
+    return ($nstreaks, @streaks);
+}
+
 
 sub my_die