Changeset 38054

Timestamp:

Mar 29, 2015, 5:45:15 AM (11 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20150112/ippScripts/scripts

Files:

• . (modified) (1 prop)
• mergedvodb_merge.pl (modified) (1 diff)
• psphot_fullforce_warp.pl (modified) (5 diffs)
• sc_prepare_camera.pl (modified) (5 diffs)
• sc_prepare_stack.pl (modified) (2 diffs)
• sc_remote_exec.pl (modified) (1 prop)
• skycalibration.pl (modified) (1 diff)
• stack_bkg_mk_mdc.pl (modified) (4 diffs)
• staticsky.pl (modified) (5 diffs)

branches/eam_branches/ipp-20150112/ippScripts/scripts/mergedvodb_merge.pl

@@ -156 +156 @@
             $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path";
         } else {
-            $merge_command = "$dvomerge -parallel $minidvodb_path into $mergedvodb_path";
+#           $merge_command = "$dvomerge -parallel $minidvodb_path into $mergedvodb_path";
+            $merge_command = "$dvomerge -matched-tables $minidvodb_path into $mergedvodb_path";
         }
         print "\n$merge_command\n";

branches/eam_branches/ipp-20150112/ippScripts/scripts/psphot_fullforce_warp.pl

@@ -124 +124 @@
 }
 
-# XXX: need to figure out whether this works or not
-# We probably want to create a specific recipe that looks at the results
-my $recipe_ppstats = 'WARPSTATS';
-my $doStats = 0;
+my $doStats = 1;
 
 print "reduction: $reduction\n";
@@ -166 +163 @@
 
 my $cmdflags = "";
+my $quality = 0;
 
 # Perform psphotFullForce
@@ -182 +180 @@
     if ($doStats) {
         $command .= " -stats $outputStats";
-        $command .= " -recipe PPSTATS $recipe_ppstats";
     }
     $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
@@ -198 +195 @@
         }
 
-        # Stats: TODO
         if ($doStats) {
             check_output($outputStats, 1);
             my $outputStatsReal = $ipprc->file_resolve($outputStats);
 
-            # measure chip stats
+            # analyze the stats file
             $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL";
             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -216 +212 @@
             chomp $cmdflags;
         }
-        my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
-
-        if (!$quality) {
-            check_output($outputSources, 1);
-        }
+        ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
+
     } else {
         print "Not executing: $command\n";
     }
+}
+
+if (!$quality) {
+    check_output($outputSources, 1);
 }
 

branches/eam_branches/ipp-20150112/ippScripts/scripts/sc_prepare_camera.pl

@@ -14 +14 @@
 use Data::Dumper;
 use File::Basename;
+use File::Temp qw( tempfile );
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -30 +31 @@
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 my $camtool    = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -88 +90 @@
 my ($workdir,$exp_tag,$reduction);
 {
+###    $command    = "$camtool -processedexp -cam_id $cam_id "; ### THIS LINE ONLY TO TEST!  
     my $command = "$camtool -pendingexp -cam_id $cam_id ";
     $command   .= " -dbname $dbname " if defined($dbname);
@@ -142 +145 @@
     my $pre_cmd_cmfs = "ls -1 ${remote_chip_path}*.cmf > ${remote_outroot}.cmflist";
     my $pre_cmd_masks= "ls -1 ${remote_chip_path}*.mk.fits > ${remote_outroot}.masklist";
+
+    # Despite the previous comment, we do care about the output of pendingimfile, as that 
+    # contains information for the exposure.
+    my ($statFile, $statName) = tempfile( "/tmp/cm.$cam_id.stats.XXXX", UNLINK => 1 );
+    foreach my $line (@$stdout_buf) {
+        print $statFile $line;
+    }
+    close $statFile;
+    
+    # parse the stats in the metadata file
+    $command = "$ppStatsFromMetadata $statName - CAMERA_EXP_IMFILE";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to perform ppStatsFromMetadata: $error_code\n");
+        exit($error_code);
+    }
+
+    my $results = '';
+    foreach my $line (@$stdout_buf) {
+        $results .= " $line";
+    }
+    chomp($results);
+
+
 
     my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use
@@ -163 +192 @@
     $camtool_post_cmd  .=  " -dbname $dbname " if defined $dbname;
     $camtool_post_cmd  .=  " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 ";
+    $camtool_post_cmd  .=  " $results "; # Add exposure information
     my $post_cmd_echo = " echo -n \"$camtool_post_cmd\" > ${remote_outroot}.dbinfo ";
     my $post_cmd_SfM  = " ppStatsFromMetadata ${remote_outroot}.stats - CAMERA_EXP_FPA >> ${remote_outroot}.dbinfo ";

branches/eam_branches/ipp-20150112/ippScripts/scripts/sc_prepare_stack.pl

@@ -23 +23 @@
 my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
 my $threads_req     = 4;                      # How many threads are we going to use?
+my $have_warps      = 0;                      # Do we have a copy of the warp if we don't find one there?
 
 my $fail_state = "prep_fail";
@@ -156 +157 @@
         my $ipp_disk;
         #  ${remote_root}/tmp/${ipp_disk}
-        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits");
-        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".fits");
-        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
-        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".wt.fits");
-        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".wt.fits");
-        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
-        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".mask.fits");
-        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".mask.fits");
-        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
-        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".cmf");
-        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".cmf");
-        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
-        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".psf");
-        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".psf");
-        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+
+        if ($have_warps != 0) { # Check to see if we have a copy, because if we do, then we can send it over if it's missing.
+            (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits");
+            ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".fits");
+            print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+
+            (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".wt.fits");
+            ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".wt.fits");
+            print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+
+            (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".mask.fits");
+            ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".mask.fits");
+            print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+
+            (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".cmf");
+            ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".cmf");
+            print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+
+            (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".psf");
+            ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".psf");
+            print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+
+        }
+        else {
+            $remote_file = &uri_local_to_remote($warp_path_base . ".fits");
+            print COMPMAP "$remote_file $remote_file\n";  # No other option, really.
+
+            $remote_file = &uri_local_to_remote($warp_path_base . ".wt.fits");
+            print COMPMAP "$remote_file $remote_file\n";  # No other option, really.
+
+            $remote_file = &uri_local_to_remote($warp_path_base . ".mask.fits");
+            print COMPMAP "$remote_file $remote_file\n";  # No other option, really.
+
+            $remote_file = &uri_local_to_remote($warp_path_base . ".cmf");
+            print COMPMAP "$remote_file $remote_file\n";  # No other option, really.
+
+            $remote_file = &uri_local_to_remote($warp_path_base . ".psf");
+            print COMPMAP "$remote_file $remote_file\n";  # No other option, really.
+
+        }
     }
 

branches/eam_branches/ipp-20150112/ippScripts/scripts/skycalibration.pl

@@ -235 +235 @@
     $command .= " -dumpconfig $configuration" if $configuration;
     $command .= " -tracedest $traceDest";
+    $command .= " -dbname $dbname" if defined $dbname;
 
     unless ($no_op) {

branches/eam_branches/ipp-20150112/ippScripts/scripts/stack_bkg_mk_mdc.pl

@@ -12 +12 @@
 
 
-my ($dbname,$stack_id,$label,$camera,$filter);
+my ($dbname,$stack_id,$sass_id,$label,$camera,$filter);
 
 my $ipprc =  PS::IPP::Config->new(); # IPP Configuration
@@ -21 +21 @@
 GetOptions(
     'stack_id=s'   => \$stack_id, 
+    'sass_id=s'    => \$sass_id,
     'label=s'      => \$label,
     'filter=s'     => \$filter,
@@ -26 +27 @@
     'camera=s'     => \$camera,
     ) or pod2usage ( 2 );
-pod2usage( -msg => "Required options; --stack_id | --label", -exitval => 3) unless
+pod2usage( -msg => "Required options; --stack_id | --label | --sass_id", -exitval => 3) unless
     ((defined $stack_id)||(defined $label));
 pod2usage( -msg => "Required options; --camera --dbname ", -exitval => 3) unless
@@ -49 +50 @@
 
 
-my $query = "SELECT DISTINCT chip_id,cam_id,path_base FROM stackRun JOIN stackInputSkyfile USING(stack_id) JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun USING (chip_id) WHERE 1 ";
+my $query = "SELECT DISTINCT chip_id,cam_id,path_base FROM stackRun JOIN stackInputSkyfile USING(stack_id) JOIN stackAssociationMap USING(stack_id) JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun USING (chip_id) WHERE 1 ";
 
 if (defined($label)) {    $query .= " AND stackRun.label = '${label}' "; }
 if (defined($stack_id)) { $query .= " AND stack_id = ${stack_id} "; }
 if (defined($filter)) {   $query .= " AND stackRun.filter = '${filter}' "; }
+if (defined($sass_id))  { $query .= " AND sass_id = ${sass_id} "; }
 
 my $cams = $db->selectall_arrayref( $query );

branches/eam_branches/ipp-20150112/ippScripts/scripts/staticsky.pl

@@ -48 +48 @@
     print "  --outroot (root)    : Output root name\n";
     print "  --reduction (class) : Reduction class\n";
+    print "  --require_sources   : Require that the stack source files exist\n";
     print "  --verbose           : Be extra verbose\n";
     print "  --no-update         : Don't update the database?\n";
@@ -60 +61 @@
 print "run staticsky.pl @ARGV\n";
 
-my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps);
+my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $require_sources, $verbose, $no_update, $no_op, $redirect, $save_temps);
 GetOptions(
     'sky_id=s'          => \$sky_id, # Diff identifier
@@ -68 +69 @@
     'outroot=s'         => \$outroot, # Output root name
     'reduction=s'       => \$reduction, # Reduction class
+    'require_sources'   => \$require_sources, # Require that the stack sources exist.
     'verbose'           => \$verbose,   # Print to stdout
     'no-update'         => \$no_update, # Don't update the database?
@@ -178 +180 @@
                 &my_die("Couldn't find input: $psfCnv",    $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv");
             }
-            &my_die("Couldn't find input: $sources",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources");
-
+            
+            my $have_sources = $ipprc->file_exists("$sources");
+            if ($require_sources) {
+                &my_die("Couldn't find input: $sources",   $sky_id, $PS_EXIT_SYS_ERROR) unless $have_sources;
+            }
+            
             print $listFile "  STACK_ID      S64  " . $stack_id  . "\n";
             print $listFile "  RAW:IMAGE     STR  " . $imageRaw  . "\n";
@@ -185 +191 @@
             print $listFile "  RAW:VARIANCE  STR  " . $weightRaw . "\n";
             print $listFile "  RAW:EXPNUM    STR  " . $expnumRaw . "\n";
-            print $listFile "  SOURCES       STR  " . $sources   . "\n";
+            if ($have_sources) {
+                print $listFile "  SOURCES       STR  " . $sources   . "\n";
+            }
             if ($needConvolvedImages) {
                 print $listFile "  CNV:IMAGE     STR  " . $imageCnv  . "\n";