Changeset 32538

Timestamp:

Oct 12, 2011, 10:57:46 AM (15 years ago)

Author:

bills

Message:

For warp destreaking check whether the tess_id for the diffs used in the
streak detection stage matches the tess_id of the warp. If not pass the diff_tess_id
to magic_destreak.pl and compute the overlap using dvoImageOverlaps

Location:

tags/ipp-20110622

Files:

• ippScripts/scripts/magic_destreak.pl (modified) (7 diffs, 1 prop)
• ippTasks/destreak.pro (modified) (2 diffs, 1 prop)
• ippTools/share/magicdstool_todestreak_camera.sql (modified) (1 diff, 1 prop)
• ippTools/share/magicdstool_todestreak_chip.sql (modified) (1 diff, 1 prop)
• ippTools/share/magicdstool_todestreak_diff.sql (modified) (2 diffs, 1 prop)
• ippTools/share/magicdstool_todestreak_raw.sql (modified) (1 diff, 1 prop)
• ippTools/share/magicdstool_todestreak_warp.sql (modified) (2 diffs, 1 prop)

tags/ipp-20110622/ippScripts/scripts/magic_destreak.pl

@@ -35 +35 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -45 +46 @@
 my ($outroot, $recoveryroot, $magicked);
 my ($replace, $release);
+my ($diff_tess_id, $mismatched_tess);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -60 +62 @@
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
            'component=s'    => \$component,  # the class_id or skycell_id
+           'mismatched_tess'=> \$mismatched_tess, # true if tess_id of input does not match tess_id of the diff used for magic
+           'diff_tess_id=s' => \$diff_tess_id, # tess_id of diffRun used to compute the streaks
            'uri=s'          => \$uri,        # uri of the input image
            'path_base=s'    => \$path_base,  # path_base of the input
@@ -106 +110 @@
 if (($stage eq "raw") or ($stage eq "chip") or $stage eq "chip_bg") {
     $class_id = $component;
-    $skycell_args = " -class_id $component";
+    $skycell_args = " -class_id";
 } elsif ($stage eq "warp" or $stage eq "warp_bg") {
     $skycell_id = $component;
-    $skycell_args = " -skycell_id $component";
+    $skycell_args = " -skycell_id";
 } elsif ($stage eq "diff") {
     $skycell_id = $component;
@@ -215 +219 @@
     my ($sfh, $skycell_list);
     if ($skycell_args) {
-        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args";
-        $command .= " -dbname $dbname" if defined $dbname;
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
-        }
-
-        my $getskycells_output = join "", @$stdout_buf;
-        if ($getskycells_output) {
-            my $metadata = $mdcParser->parse($getskycells_output) or
-                &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-            my $skycells = parse_md_list($metadata) or
-                    &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-            ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
-
-            foreach my $skycell (@$skycells) {
-                my $skycell_uri;
-                if ($skycell->{data_state} eq "full" and $skycell->{quality} = 0) {
-                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
-                } else {
-                    # diff run must have been cleaned up, need to create this skycell file on the fly
-                    if (!defined $temp_dir ) {
-                        $temp_dir = tempdir( CLEANUP => !$save_temps);
+        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells";
+        $command .= " $skycell_args";
+        my @diff_components;
+        if ($mismatched_tess) {
+            # tessellation for this skycell does not match the tessellation used for the magic analysis
+            # used to calculate the streaks file
+            @diff_components = get_overlaps($path_base, $component, $diff_tess_id);
+        } else {
+            @diff_components = ($component);
+        }
+
+        # hash of uris by skycell.
+        my %diff_skycells;
+        foreach my $component (@diff_components) {
+            my $this_command = "$command $component";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $this_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
+            }
+
+            my $getskycells_output = join "", @$stdout_buf;
+            if ($getskycells_output) {
+                my $metadata = $mdcParser->parse($getskycells_output) or
+                    &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+                my $skycells = parse_md_list($metadata) or
+                        &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+                foreach my $skycell (@$skycells) {
+                    my $skycell_uri;
+                    if ($skycell->{data_state} eq "full" and $skycell->{quality} == 0) {
+                        $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                        $diff_skycells{$skycell_id} = $skycell_uri;
+                    } else {
+                        # diff run must have been cleaned up, need to create this skycell file on the fly
+                        if (!defined $temp_dir ) {
+                            $temp_dir = tempdir( CLEANUP => !$save_temps);
+                        }
+                        my $skycell_id = $skycell->{skycell_id};
+                        if (!$diff_skycells{$skycell_id}) {
+                            $skycell_uri = "$temp_dir/$skycell_id";
+                            $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
+                                &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+                        }
+                        $diff_skycells{$skycell_id} = $skycell_uri;
                     }
-                    my $skycell_id = $skycell->{skycell_id};
-                    $skycell_uri = "$temp_dir/$skycell_id";
-                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
-                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
                 }
-                print $sfh "$skycell_uri\n";
-            }
-            close $sfh
-        }
+            }
+        }
+        # write the skycell list file
+        ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
+        foreach my $skycell_id (keys %diff_skycells) {
+            print $sfh $diff_skycells{$skycell_id} . "\n";
+        }
+        close $sfh
     }
 
@@ -470 +494 @@
 
 my $statsFlags;
-if ($stage ne "camera") {
+if (!$no_op and $stage ne "camera") {
     file_check($statsFile);
     {
@@ -590 +614 @@
 }
 
+            # @diff_components = get_overlaps($path_base, $component, $tess_id, $diff_tess_id);
+sub get_overlaps {
+    my $path_base = shift;
+    my $component = shift;
+    my $diff_tess_id = shift;
+
+    my $tess_dir = $ipprc->tessellation_catdir( $diff_tess_id ); 
+    my $catdir = $ipprc->convert_filename_absolute($tess_dir);
+
+    my $cmf = $ipprc->file_resolve("$path_base.cmf");
+    &my_die("Unable to resolve $path_base.$cmf", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless $cmf;
+
+    my $command = "$dvoImageOverlaps -D CATDIR $catdir -accept-astrom $cmf";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dvoImageOverlaps $error_code", $magic_ds_id, $component,
+            $error_code);
+    }
+    my @list;
+    my $output = join "\n", @$stdout_buf;
+    if ($output) {
+        foreach my $line (split "\n", $output) {
+            my (undef, undef, $skycell_id) = split " ", $line;
+            push @list, $skycell_id;
+        }
+    }
+    return @list
+}
+
 
 sub my_die

tags/ipp-20110622/ippTasks/destreak.pro

@@ -206 +206 @@
     book getword magicToDS $pageName magicked -var MAGICKED
     book getword magicToDS $pageName dbname -var DBNAME
-
+    book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID
+    book getword magicToDS $pageName mismatched_tess -var MISMATCHED_TESS
 
     substr $COMPONENT 0 3 COMP_HEAD
@@ -224 +225 @@
     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE
 
+    if ($MISMATCHED_TESS) 
+        book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID
+        $run = $run --mismatched_tess --diff_tess_id $DIFF_TESS_ID
+    end
     add_standard_args run
 

tags/ipp-20110622/ippTools/share/magicdstool_todestreak_camera.sql

@@ -14 +14 @@
     stage_id,
     'exposure' AS component,
+    0 AS mismatched_tess,
     CAST(NULL AS CHAR(255)) AS uri,
     camProcessedExp.path_base,

tags/ipp-20110622/ippTools/share/magicdstool_todestreak_chip.sql

@@ -14 +14 @@
     stage_id,
     class_id AS component,
+    0 AS mismatched_tess,
     chipProcessedImfile.uri,
     chipProcessedImfile.path_base,

tags/ipp-20110622/ippTools/share/magicdstool_todestreak_diff.sql

@@ -15 +15 @@
     diffRun.magicked,
     diffSkyfile.skycell_id AS component,
+    0 AS mismatched_tess,
     CAST(NULL AS CHAR(255)) AS uri,
     diffSkyfile.path_base,
@@ -63 +64 @@
     diffRun.magicked,
     diffSkyfile.skycell_id AS component,
+    0 AS mismatched_tess,
     CAST(NULL AS CHAR(255)) AS uri,
     diffSkyfile.path_base,

tags/ipp-20110622/ippTools/share/magicdstool_todestreak_raw.sql

@@ -14 +14 @@
     rawExp.magicked,
     class_id as component,
+    0 AS mismatched_tess,
     rawImfile.uri AS uri,
     -- XXX: replace this with rawImfile.path_base once it exists

tags/ipp-20110622/ippTools/share/magicdstool_todestreak_warp.sql

@@ -14 +14 @@
     warpRun.magicked,
     warpSkyfile.skycell_id as component,
+    diffRun.tess_id != warpRun.tess_id AS mismatched_tess,
+    diffRun.tess_id as diff_tess_id,
     warpSkyfile.uri,
     warpSkyfile.path_base,
@@ -27 +29 @@
 JOIN magicMask USING (magic_id)
 JOIN magicRun USING (magic_id)
+JOIN diffRun USING(diff_id)
 JOIN warpRun ON warp_id = stage_id
 JOIN warpSkyfile USING(warp_id)