Changeset 35236 for branches/eam_branches/ipp-20130207/ippScripts

Timestamp:

Mar 5, 2013, 2:24:05 PM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130207

Files:

• . (modified) (1 prop)
• ippScripts/scripts/automate_stacks.pl (modified) (3 diffs)
• ippScripts/scripts/background_chip.pl (modified) (7 diffs)
• ippScripts/scripts/background_warp.pl (modified) (3 diffs)
• ippScripts/scripts/camera_exp.pl (modified) (1 diff)
• ippScripts/scripts/chip_imfile.pl (modified) (1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/diff_skycell.pl (modified) (1 diff)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/ipp_cleanup.pl (modified) (2 diffs)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (7 diffs, 1 prop)
• ippScripts/scripts/skycalibration.pl (modified) (1 diff)
• ippScripts/scripts/warp_skycell.pl (modified) (1 diff, 1 prop)

branches/eam_branches/ipp-20130207/ippScripts/scripts/automate_stacks.pl

@@ -117 +117 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-          -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks",
+          -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks --dbname",
           -exitval => 3,
           ) unless
@@ -124 +124 @@
     defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or
     defined $test_mode or defined $clean_old or defined $check_mode or
-    defined $confirm_stacks or defined $burntool_stats;
+    defined $confirm_stacks or defined $burntool_stats or defined $dbname;
 
 # Configurable parameters from our config file.
@@ -282 +282 @@
 unless(defined($camera)) {
     $camera = 'GPC1';
-}
-unless(defined($dbname)) {
-    $dbname = 'gpc1';
 }
 

branches/eam_branches/ipp-20130207/ippScripts/scripts/background_chip.pl

@@ -43 +43 @@
 # Parse the command-line arguments
 my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose,
-     $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $magicked );
+     $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $cam_path_base, $magicked );
 GetOptions(
     'chip_bg_id=s'      => \$chip_bg_id,    # chipBackgroundRun identifier
@@ -53 +53 @@
     'threads=s'         => \$threads,   # Number of threads to use
     'chip_path_base=s'  => \$chip_path_base, # optional chip_path_base
+    'cam_path_base=s'   => \$cam_path_base, # optional camera stage path_base
     'magicked=s'        => \$magicked,  # magicked status of input
     'verbose'           => \$verbose,   # Print to stdout
@@ -94 +95 @@
 my $dump_config;
 if ($chip_path_base and $no_update) {
+    # XXX: this path through the code has not been excercized in awhile and probably does not work
+
     # we are running outside of a chip_bg_run (perhaps by the postage stamp server) don't dump config
     # or do stats. Get path to input from command line.
@@ -119 +122 @@
     my $input = $$inputs[0];    # Input of interest
     $in_path = $input->{path_base};
+    $cam_path_base = $input->{cam_path_base};
     $magicked = $input->{magicked};
 }
 
 my $in_image = $ipprc->filename("PPIMAGE.CHIP", $in_path, $class_id);
-my $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id);
+my $in_mask;
+if ($cam_path_base) {
+    $in_mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
+} else {
+    $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id);
+}
 my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id);
 my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id);
@@ -177 +186 @@
 
 my $apply_auxiliary_mask = 1;
+# XXX Temporary lookup of auxiliary mask file from static location.
+# At some point I'll convert this to register the masks in the database and use detselect
 if ($apply_auxiliary_mask) {
     my @auxmask_start_date = qw(
@@ -238 +249 @@
     print "Auxililary mask file is $auxmask\n";
 }
-    
 
 
@@ -272 +282 @@
     &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats);
 
-    my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_CHIP";
+    my $resolved_stats = $ipprc->file_resolve($out_stats);
+    my $command = "$ppStatsFromMetadata $resolved_stats - BACKGROUND_CHIP";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);

branches/eam_branches/ipp-20130207/ippScripts/scripts/background_warp.pl

@@ -119 +119 @@
 my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id";
 
-# XXX: get this from recipe
-my $alternate_astrometry_label = 'M31.test.20130206.bgsub';
-# XXX: get exp_id from command line
-my $exp_id = 196758;
-
 if (!defined $imageName) {
     # go find our inputs
-    my $astrom_path_base;
-    if (defined $alternate_astrometry_label) {
-        my $command = "$camtool -processedexp -exp_id $exp_id -label $alternate_astrometry_label";
-        $command .= " -dbname $dbname" if $dbname;
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform camtool -processedexp: $error_code", $warp_bg_id, $error_code);
-        }
-        my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or
-            &my_die("Unable to parse metadata config doc", $warp_bg_id, $PS_EXIT_PROG_ERROR);
-        my $entry = $entries->[0];
-        $astrom_path_base = $entry->{path_base};
-    }
-
     my ($imageFile, $maskFile, $weightFile);
     ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX",  UNLINK => !$save_temps);
@@ -174 +153 @@
         $magicked = $file->{magicked};
 
-        if (!$astrom_path_base) {
-            my $cam_path = $file->{cam_path_base};
-            my $astrom = $ipprc->filename($astromSource, $cam_path);
-            &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry;
-            $astrometry = $astrom;
-        } elsif (!$astrometry) {
-            # using alternate astrometry
-            $astrometry = $ipprc->filename($astromSource, $astrom_path_base);
-        }
+        my $cam_path = $file->{cam_path_base};
+        my $astrom = $ipprc->filename($astromSource, $cam_path);
+        &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry;
+        $astrometry = $astrom;
     }
     close $imageFile;
@@ -238 +212 @@
     &my_die("Couldn't find expected output file: $out_stats", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats);
 
-    my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_WARP";
+    my $real_stats = $ipprc->file_resolve($out_stats);
+
+    my $command = "$ppStatsFromMetadata $real_stats - BACKGROUND_WARP";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);

branches/eam_branches/ipp-20130207/ippScripts/scripts/camera_exp.pl

@@ -379 +379 @@
             check_output($bkgModel, $replicateOutputs);
         }
+        $cmdflags = $cmdflags . " -background_model 1 ";
     }
 }

branches/eam_branches/ipp-20130207/ippScripts/scripts/diff_skycell.pl

@@ -299 +299 @@
 my $jpeg1Name = prepare_output("PPSUB.OUTPUT.JPEG1", $outroot, 1);
 my $jpeg2Name = prepare_output("PPSUB.OUTPUT.JPEG2", $outroot, 1);
+
+# do we always need to prepare this file?
+if ($reduction ne 'NOCONVDIFF') {
+    my $refConv = prepare_output("PPSUB.REF.CONV", $outroot, 1);
+}
 
 my ($inverseName, $inverseMask, $inverseVariance, $inverseSources);

branches/eam_branches/ipp-20130207/ippScripts/scripts/ipp_cleanup.pl

@@ -1957 +1957 @@
             addFilename (\@files, "PPBACKGROUND.OUTPUT", $path_base, $class_id);
             addFilename (\@files, "PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id);
+            addFilename (\@files, "PPBACKGROUND.OUTPUT.VARIANCE", $path_base, $class_id);
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode
@@ -2100 +2101 @@
             addFilename (\@files, "PSWARP.OUTPUT", $path_base, $skycell_id);
             addFilename (\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
+            addFilename (\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode

branches/eam_branches/ipp-20130207/ippScripts/scripts/publish_file.pl

@@ -92 +92 @@
 
 my $comment;                    # Comment for exposure
+my $exp_name_1;                    # Name of exposure 1
+my $exp_name_2;                    # Name of exposure 2
 if ($stage eq 'camera') {
     my $command =  "camtool -processedexp -cam_id $stage_id";
@@ -168 +170 @@
     my %negative;               # Data for negative diff detections
     foreach my $comp ( @$components ) {
-        my $path_base = $comp->{path_base}; # Base name for file
+        my $path_base = $comp->{path_base}; # Base name for file
         if (!$need_magic and $comp->{magicked}) {
             # This client is authorized to receive uncensored detections
@@ -178 +180 @@
         print "Warning: mis-matched comments\n" if defined $comment and $comment ne $comp->{comment};
         $comment = $comp->{comment} unless defined $comment;
+        $exp_name_1 = $comp->{exp_name_1} unless defined $exp_name_1;
+        $exp_name_2 = $comp->{exp_name_2} unless defined $exp_name_2;
 
         (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time};
@@ -192 +196 @@
         &my_die("input file does not exist: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$ipprc->file_exists($filename);
         my $resolved = $ipprc->file_resolve($filename);
-        
+
         &my_die("unable to resolve input file: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$resolved;
 
         $filename = $resolved;
 
-        my $cam_id = $comp->{cam_id_1};
-        #print "Getting info from camera stage $cam_id\n";
-        my $cam_command =  "$camtool -processedexp -cam_id $cam_id";
-        $cam_command .= " -dbname $dbname" if defined $dbname;
-        my ( $cam_success, $cam_error_code, $cam_full_buf, $cam_stdout_buf, $cam_stderr_buf ) =
-            run(command => $cam_command, verbose => $verbose);
-        &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $cam_success;
-        my $cam_metadata = $mdcParser->parse(join "", @$cam_stdout_buf) or
-            &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
-        my $cam_components = parse_md_list($cam_metadata) or
-            &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
-        &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$cam_components > 1;
-        my $cam_comp = $$cam_components[0];
-
-        my $exp_id = $comp->{exp_id_1};
-        #print "Getting info from raw stage $exp_id\n";
-        my $reg_command =  "$regtool -processedimfile -exp_id $exp_id";
-        $reg_command .= " -dbname $dbname" if defined $dbname;
-        my ( $reg_success, $reg_error_code, $reg_full_buf, $reg_stdout_buf, $reg_stderr_buf ) =
-            run(command => $reg_command, verbose => $verbose);
-        &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $reg_success;
-        my $reg_metadata = $mdcParser->parse(join "", @$reg_stdout_buf) or
-            &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
-        my $reg_components = parse_md_list($reg_metadata) or
-            &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
-        my $reg_comp = $$reg_components[0];
-
-        # Now get the difftype value from diff->{diff_mode}
-        my $difftype = "";
-        if ($comp->{diff_mode} == 1) {
-            $difftype = "WW";
-        } elsif ($comp->{diff_mode} == 2) {
-            $difftype = "WS";
-        } elsif ($comp->{diff_mode} == 3) {
-            $difftype = "SW"; # Not used yet
-        } elsif ($comp->{diff_mode} == 4) {
-            $difftype = "SS";
-        } else {
-            $difftype = "Unsupported diff_mode value: [" . $comp->{diff_mode} . "]";
-        }
+        my $cam_id = $comp->{cam_id_1};
+        #print "Getting info from camera stage $cam_id\n";
+        my $cam_command =  "$camtool -processedexp -cam_id $cam_id";
+        $cam_command .= " -dbname $dbname" if defined $dbname;
+        my ( $cam_success, $cam_error_code, $cam_full_buf, $cam_stdout_buf, $cam_stderr_buf ) =
+            run(command => $cam_command, verbose => $verbose);
+        &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $cam_success;
+        my $cam_metadata = $mdcParser->parse(join "", @$cam_stdout_buf) or
+            &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
+        my $cam_components = parse_md_list($cam_metadata) or
+            &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
+        &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$cam_components > 1;
+        my $cam_comp = $$cam_components[0];
+
+        my $exp_id = $comp->{exp_id_1};
+        #print "Getting info from raw stage $exp_id\n";
+        my $reg_command =  "$regtool -processedimfile -exp_id $exp_id";
+        $reg_command .= " -dbname $dbname" if defined $dbname;
+        my ( $reg_success, $reg_error_code, $reg_full_buf, $reg_stdout_buf, $reg_stderr_buf ) =
+            run(command => $reg_command, verbose => $verbose);
+        &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $reg_success;
+        my $reg_metadata = $mdcParser->parse(join "", @$reg_stdout_buf) or
+            &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
+        my $reg_components = parse_md_list($reg_metadata) or
+            &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
+        my $reg_comp = $$reg_components[0];
+
+        # Now get the difftype value from diff->{diff_mode}
+        my $difftype = "";
+        if ($comp->{diff_mode} == 1) {
+            $difftype = "WW";
+        } elsif ($comp->{diff_mode} == 2) {
+            $difftype = "WS";
+        } elsif ($comp->{diff_mode} == 3) {
+            $difftype = "SW"; # Not used yet
+        } elsif ($comp->{diff_mode} == 4) {
+            $difftype = "SS";
+        } else {
+            $difftype = "Unsupported diff_mode value: [" . $comp->{diff_mode} . "]";
+        }
 
         my $data = { zp => $zp,
@@ -250 +254 @@
                      output_format => $comp->{output_format},
                      direction => 1,
-                     comment => $reg_comp->{comment},
-                     obsmode => $reg_comp->{obs_mode},
-                     difftype => $difftype,
-                     sky => $cam_comp->{bg},
-                     shutoutc => $reg_comp->{dateobs},
+                     comment => $reg_comp->{comment},
+                     obsmode => $reg_comp->{obs_mode},
+                     difftype => $difftype,
+                     sky => $cam_comp->{bg},
+                     shutoutc => $reg_comp->{dateobs},
         };
 
-        #warn("Checking for positive");
+        #warn("Checking for positive");
         diff_check(\%positive, $data, "positive");
 
@@ -288 +292 @@
                          diff_id => $comp->{diff_id},
                          direction => 0,
-                         comment => $reg_comp->{comment},
-                         obsmode => $reg_comp->{obs_mode},
-                         difftype => $difftype,
-                         sky => $cam_comp->{bg},
-                         shutoutc => $reg_comp->{dateobs},
+                         comment => $reg_comp->{comment},
+                         obsmode => $reg_comp->{obs_mode},
+                         difftype => $difftype,
+                         sky => $cam_comp->{bg},
+                         shutoutc => $reg_comp->{dateobs},
             };
 
-            #warn("Checking for negative");
+            #warn("Checking for negative");
             diff_check(\%negative, $data, "negative");
 
@@ -326 +330 @@
     my $command = "$dsreg --add pub.$pub_id.$stage.$stage_id --copy --abspath --product $product --type $dsType --list $dsFileName";
     $command .= " --ps0 \"$comment\"" if defined $comment;
+    $command .= " --ps1 \"$exp_name_1\"" if defined $exp_name_1;
+    $command .= " --ps2 \"$exp_name_2\"" if defined $exp_name_2;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

branches/eam_branches/ipp-20130207/ippScripts/scripts/skycalibration.pl

@@ -231 +231 @@
     $command .= " -F PSASTRO.OUTPUT $output_sources_filerule";
     $command .= " -stats $stats";
-    $command .= " -recipe PPSTATS CAMSTATS";
+    # $command .= " -recipe PPSTATS CAMSTATS";
+    $command .= " -recipe PPSTATS SKYCALSTATS";
     $command .= " -dumpconfig $configuration" if $configuration;
     $command .= " -tracedest $traceDest";

branches/eam_branches/ipp-20130207/ippScripts/scripts/warp_skycell.pl

@@ -245 +245 @@
     print $weightFile "$weight\n";
     my $bkg;
-    if ($doBackground) {
-        $bkg    = $ipprc->filename("PSPHOT.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id});
+    if (($doBackground)) {
+        # &&($imfile->{background_model} == 1)
+        if ($imfile->{cam_background_model}) {
+            $bkg    = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{cam_path_base}, $imfile->{class_id});
+        }
+        else {
+            $bkg    = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id});
+        }
         print $bkgFile "$bkg\n";
     }