Changeset 33415 for branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak.pl

Timestamp:

Mar 5, 2012, 5:19:48 PM (14 years ago)

Author:

mhuber

Message:

merging latest r33407 trunk changes to meh_branches/ppstack_test

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (10 diffs, 1 prop)

branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak.pl

@@ -35 +35 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -45 +46 @@
 my ($outroot, $recoveryroot, $magicked);
 my ($replace, $release);
+my ($diff_tess_id, $mismatched_tess);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -60 +62 @@
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
            'component=s'    => \$component,  # the class_id or skycell_id
+           'mismatched_tess'=> \$mismatched_tess, # true if tess_id of input does not match tess_id of the diff used for magic
+           'diff_tess_id=s' => \$diff_tess_id, # tess_id of diffRun used to compute the streaks
            'uri=s'          => \$uri,        # uri of the input image
            'path_base=s'    => \$path_base,  # path_base of the input
@@ -95 +99 @@
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
+
+if ($logfile) {
+    $ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR );
+    print "\n\n";
+    print "Continuing script $0 on $host at $date\n\n";
+}
 
 $cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL');
@@ -106 +115 @@
 if (($stage eq "raw") or ($stage eq "chip") or $stage eq "chip_bg") {
     $class_id = $component;
-    $skycell_args = " -class_id $component";
+    $skycell_args = " -class_id";
 } elsif ($stage eq "warp" or $stage eq "warp_bg") {
     $skycell_id = $component;
-    $skycell_args = " -skycell_id $component";
+    $skycell_args = " -skycell_id";
 } elsif ($stage eq "diff") {
     $skycell_id = $component;
@@ -176 +185 @@
 }
 
-my $statsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.stats";
+# XXX: create a DESTREAK file rule. For now just steal one that will work
+my $error;
+my $statsFile = $ipprc->prepare_output("PSASTRO.STATS", "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component", undef, 1, \$error) or &my_die("failed to prepare output for stats", $magic_ds_id, $component, $error);
 
 my $backup_path_base;
@@ -215 +226 @@
     my ($sfh, $skycell_list);
     if ($skycell_args) {
-        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args";
-        $command .= " -dbname $dbname" if defined $dbname;
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
-        }
-
-        my $getskycells_output = join "", @$stdout_buf;
-        if ($getskycells_output) {
-            my $metadata = $mdcParser->parse($getskycells_output) or
-                &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-            my $skycells = parse_md_list($metadata) or
-                    &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-            ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
-
-            foreach my $skycell (@$skycells) {
-                my $skycell_uri;
-                if ($skycell->{data_state} eq "full" and $skycell->{quality} = 0) {
-                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
-                } else {
-                    # diff run must have been cleaned up, need to create this skycell file on the fly
-                    if (!defined $temp_dir ) {
-                        $temp_dir = tempdir( CLEANUP => !$save_temps);
+        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells";
+        $command .= " $skycell_args";
+        $command .= " -dbname $dbname" if $dbname;
+        my @diff_components;
+        if ($mismatched_tess) {
+            # tessellation for this skycell does not match the tessellation used for the magic analysis
+            # used to calculate the streaks file
+            @diff_components = get_overlaps($path_base, $component, $diff_tess_id);
+        } else {
+            @diff_components = ($component);
+        }
+
+        # hash of uris by skycell.
+        my %diff_skycells;
+        foreach my $component (@diff_components) {
+            my $this_command = "$command $component";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $this_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
+            }
+
+            my $getskycells_output = join "", @$stdout_buf;
+            if ($getskycells_output) {
+                my $metadata = $mdcParser->parse($getskycells_output) or
+                    &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+                my $skycells = parse_md_list($metadata) or
+                        &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+                foreach my $skycell (@$skycells) {
+                    my $skycell_uri;
+                    $skycell->{quality} = undef;
+                    # XXXXXXXX remake all skycells to avoid dead instances (ipp064)
+                    if (0 and $skycell->{data_state} eq "full" and $skycell->{quality} == 0) {
+                        $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                        my $skycell_id = $skycell->{skycell_id};
+                        $diff_skycells{$skycell_id} = $skycell_uri;
+                    } else {
+                        # diff run must have been cleaned up, need to create this skycell file on the fly
+                        if (!defined $temp_dir ) {
+                            $temp_dir = tempdir( CLEANUP => !$save_temps);
+                        }
+                        my $skycell_id = $skycell->{skycell_id};
+                        if (!$diff_skycells{$skycell_id}) {
+                            $skycell_uri = "$temp_dir/$skycell_id";
+                            $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
+                                &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+                        }
+                        $diff_skycells{$skycell_id} = $skycell_uri;
                     }
-                    my $skycell_id = $skycell->{skycell_id};
-                    $skycell_uri = "$temp_dir/$skycell_id";
-                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
-                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
                 }
-                print $sfh "$skycell_uri\n";
-            }
-            close $sfh
-        }
+            }
+        }
+        # write the skycell list file
+        ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
+        foreach my $skycell_id (keys %diff_skycells) {
+            print $sfh $diff_skycells{$skycell_id} . "\n";
+        }
+        close $sfh
     }
 
@@ -360 +395 @@
         $command .= " -chip_mask $ch_mask" if defined $ch_mask;
         $command .= " -weight $weight" if defined $weight;
-        if ((defined($sources))&&($ipprc->file_exists($sources))) {
-            $command .= " -sources $sources" if defined $sources;
-        }
-        else {
-            print "Did not add sources because they do not appear to exist. This may be an error.\n";
-        }
+        $command .= " -sources $sources" if defined $sources;
         $command .= " -skycelllist $skycell_list" if defined $skycell_list;
         $command .= " -replace" if $replace;
@@ -475 +505 @@
 
 my $statsFlags;
-if ($stage ne "camera") {
+if (!$no_op and $stage ne "camera") {
     file_check($statsFile);
     {
@@ -595 +625 @@
 }
 
+            # @diff_components = get_overlaps($path_base, $component, $tess_id, $diff_tess_id);
+sub get_overlaps {
+    my $path_base = shift;
+    my $component = shift;
+    my $diff_tess_id = shift;
+
+    my $tess_dir = $ipprc->tessellation_catdir( $diff_tess_id ); 
+    my $catdir = $ipprc->convert_filename_absolute($tess_dir);
+
+    my $cmf = $ipprc->file_resolve("$path_base.cmf");
+    &my_die("Unable to resolve $path_base.$cmf", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless $cmf;
+
+    my $command = "$dvoImageOverlaps -D CATDIR $catdir -accept-astrom $cmf";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dvoImageOverlaps $error_code", $magic_ds_id, $component,
+            $error_code);
+    }
+    my @list;
+    my $output = join "\n", @$stdout_buf;
+    if ($output) {
+        foreach my $line (split "\n", $output) {
+            my (undef, undef, $skycell_id) = split " ", $line;
+            push @list, $skycell_id;
+        }
+    }
+    return @list
+}
+
 
 sub my_die