Changeset 31587 for branches/eam_branches/ipp-20110505/ippScripts

Timestamp:

May 29, 2011, 11:26:31 AM (15 years ago)

Author:

eugene

Message:

merging changes from trunk

Location:

branches/eam_branches/ipp-20110505/ippScripts/scripts

Files:

• camera_exp.pl (modified) (1 diff)
• dist_advancerun.pl (modified) (1 diff)
• dist_bundle.pl (modified) (6 diffs)
• ipp_apply_burntool_single.pl (modified) (1 prop)
• lap_science.pl (modified) (21 diffs)
• receive_file.pl (modified) (2 diffs)
• warp_skycell.pl (modified) (4 diffs)

branches/eam_branches/ipp-20110505/ippScripts/scripts/camera_exp.pl

@@ -153 +153 @@
     }
     chomp $cmdflags;
+
+    { # Determine if FWHM is too large to bother continuing.
+        my $command = "$ppConfigDump -camera $camera -dump-camera -";
+        my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
+        }
+        my $cameraConfig = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
+
+        my $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM');
+        if ($maxFWHM) {
+            my $expFWHM;
+            ($expFWHM) = $cmdflags =~ /-fwhm_major (\d+)/;      
+            
+            if ($expFWHM > $maxFWHM) {
+                print "Setting quality to 4007 due to large FWHM: exposure: $expFWHM  maximum: $maxFWHM\n";
+                $cmdflags .= " -quality 4007 "; # This corresponds to PSASTRO_ERR_DATA
+                $no_op = 1;
+            }
+        }
+    }
 }
 

branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_advancerun.pl

@@ -114 +114 @@
     $tool_cmd = "$staticskytool -sky_id";
     $list_mode = "-result";
-    $component_key = "";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
     $tool_cmd = "$difftool -diff_id";

branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_bundle.pl

@@ -62 +62 @@
 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
 my $bgtool   = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $staticskytool   = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
 my $file_cmd   = can_run('file') or (warn "can't find program file" and $missing_tools = 1);
 my $zcat   = can_run('zcat') or (warn "can't find program zcat" and $missing_tools = 1);
@@ -106 +107 @@
     defined $stage_id and
     defined $component and
-    defined $exp_type and
     defined $path_base and
     defined $outroot;
 
+if ($stage eq 'raw' and !$clean and !$no_magic) {
+    
+    # for raw stage need to have exposure type defined and if the type is OBJECT we need
+    # a chip_path_base so we can find the chip mask file
+    if (!defined $exp_type or ($exp_type eq 'OBJECT' and !defined $chip_path_base)) {
+        pod2usage( -msg => "Required options: --chip_path_base --exp_type for raw stage", -exitval => 3);
+    }
+}
+
 $ipprc->redirect_output($logfile) if $logfile;
 
-if ($stage eq 'raw' and !$clean and !$no_magic) {
-    # need to be able to find chip mask file
-    if ($exp_type eq 'OBJECT' and !defined $chip_path_base) {
-        pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3);
-    }
-}
-
 $ipprc->define_camera($camera);
 
 $ipprc->outroot_prepare($outroot);
 
+my $num_sky_inputs;
+if ($stage eq 'sky') {
+    $num_sky_inputs = get_num_sky_inputs($stage_id);
+}
 # Get the list of data products for this component
 # note: We my_die in get_file_list if something goes wrong.
 
-my $file_list = get_file_list($stage, $component, $path_base, $clean);
+my $file_list = get_file_list($stage, $component, $path_base, $clean, $num_sky_inputs);
 
 if (($stage ne 'raw') and ($stage ne 'fake') and !$poor_quality) {
@@ -184 +190 @@
         # it must exist)
         next if $poor_quality;
+
+        # skip file stats file. Due to a bug the update process destroys them sometimes
+        # XXX: perhaps only do this for stages where we know that this happens
+        next if $file_rule =~ /STATS/;
 
         &my_die("failed to resolve  $file_name", $component, $PS_EXIT_DATA_ERROR);
@@ -480 +490 @@
     my $path_base = shift;
     my $clean = shift;
+    my $num_sky_inputs = shift;
 
     my @file_list;
@@ -531 +542 @@
         $config_file_rule = "PPSTACK.CONFIG";
     } elsif ($stage eq "sky") {
-        $config_file_rule = "PSPHOT.STACK.CONFIG";
+        if ($num_sky_inputs > 1) {
+            $config_file_rule = "PSPHOT.STACK.CONFIG";
+        } else {
+            $config_file_rule = "PSPHOT.SKY.CONFIG";
+        }
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -627 +642 @@
     return \@file_list;
 }
+sub get_num_sky_inputs {
+    my $sky_id = shift;
+
+    my $command = "$staticskytool -inputs -sky_id $sky_id -simple";
+    $command .= " -dbname $dbname" if $dbname;
+    $command .= " | wc";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $component, $error_code);
+    }
+    my ($num_inputs, $words, $chars) = split " ", (join "", @$stdout_buf);
+    if (!$num_inputs) {
+        $num_inputs = "undefined" if !defined $num_inputs;
+        &my_die("unexpected number of static sky inputs $num_inputs",  $PS_EXIT_PROG_ERROR, $component, $error_code);
+    }
+
+    return $num_inputs;
+}
 
 sub my_die

branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl

@@ -99 +99 @@
     my $status = queue_chips($lap_id);
 
-    if ($status) {
+    if (!$status) { # This is the culprit.
         my $command = "$laptool -updaterun -lap_id $lap_id";
         $command .= " -dbname $dbname " if defined $dbname;
@@ -120 +120 @@
     # This is a puzzler... chiptool doesn't actually return a useful metadata.  We'll just scrape it from the database for now.
 
-    my $command = "$chiptool -listrun -exp_id $exp_id -label $label -data_group $data_group";
+    my $command = "$chiptool -listrun -pstamp_order -exp_id $exp_id -label $label -data_group $data_group";
     $command .= " -dbname $dbname " if defined $dbname;
 
@@ -236 +236 @@
         my $comment = $exposure->{comment};
 
+        # This is a hack to fix old exposures that have no object.
+        unless(defined($comment)) {
+            $comment = '';
+        }
+        if ((!defined($object))||($object eq 'NULL')||($object eq '')) {
+            if ($comment =~ /3pi_/) {
+                $object = $comment;
+                $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/;
+            }
+            elsif ($comment =~ / ps1_/) {
+                $object = $comment;
+                $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/;
+            }
+            else {
+                $exposure->{data_state} = 'new';
+                $exposure->{pairwise} = 0;
+                $exposure->{private} = 1;
+                $exposure->{pair_id} = 9223372036854775807;
+                update_this_exposure($exposure);
+                $counter++;
+                next;
+            }
+        }
+
+
         if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id
             if (($pairwise) && !($pair_id)) {
@@ -255 +280 @@
             $counter++;
         }
+        print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n";
     }
 
@@ -262 +288 @@
         foreach my $comment (keys %{ $matching{$object} }) {
             push @exp_ids_to_diff, $matching{$object}{$comment};
+            print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n";
         }
         @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff;
@@ -275 +302 @@
             my $exp_A = ${ $exposures }[$indexing{$exp_id_A}];
             my $exp_B = ${ $exposures }[$indexing{$exp_id_B}];
-
+            print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n";
             $exp_A->{pairwise} = 1;
             $exp_A->{private} = 0;
@@ -292 +319 @@
     # Scan all exposures, and ensure that pairwise and private are set correctly
     foreach my $exposure (@$exposures) { 
+        print "YY: $exposure\n";
         if ($exposure->{pairwise} && !($exposure->{pair_id})) {
             $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error.
@@ -299 +327 @@
         }
 
-        $exposure = update_this_exposure($exposure);
-    }
-    return(1);
+        update_this_exposure($exposure);
+    }
+    return(0);
 }
 
@@ -432 +460 @@
         my $companion;
 
-        if ($exposure->{pair_id}) { # Load companion exposure information
-            if (exists($match_hash{$exposure->{chip_id}})) {
+        if ($exposure->{pairwise}) {
+            # Load companion exposure information
+            if (($exposure->{pair_id})&&(exists($match_hash{$exposure->{chip_id}}))) {
                 $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match!
+            }
+            else { # We claimed to be pairwise, but do not have a valid pair_id.
+                $exposure->{pairwise} = 0;
+                $exposure->{private} = 1;
+                &update_this_exposure($exposure);
             }
         }
@@ -446 +480 @@
             $exposure = remake_this_exposure($exposure);
         }
-        if ($exposure->{cam_quality}) {
-            $needs_qstack = 1;
-            $needs_something_private = 1;
-            if ($companion) {
-                $companion->{private} = 1;
-                $companion->{pairwise} = 0;
-                &update_this_exposure($companion);
+        # Do quality checks here
+        my $is_bad_quality = 0;
+        if ((defined($exposure->{chipRun_state}))&&($exposure->{chipRun_state} eq 'full')&&
+            ($exposure->{chip_component_count} > 0)&&($exposure->{chip_bad_quality} / $exposure->{chip_component_count} > 0.05)) {
+            printf("QUALITY: $exposure->{exp_id} has bad chip quality: %d / %d\n",
+                   $exposure->{chip_bad_quality} , $exposure->{chip_component_count});
+            $is_bad_quality = 1;
+        }
+        elsif ((defined($exposure->{camRun_state}))&&($exposure->{camRun_state} eq 'full')&&
+               ($exposure->{cam_bad_quality} / $exposure->{cam_component_count} > 0)) {
+            printf("QUALITY: $exposure->{exp_id} has bad cam quality: %d / %d\n",
+                   $exposure->{cam_bad_quality} , $exposure->{cam_component_count});
+            $is_bad_quality = 1;
+        }
+        elsif ((defined($exposure->{warpRun_state}))&&($exposure->{warpRun_state} eq 'full')&&
+               ($exposure->{warp_bad_quality} / $exposure->{warp_component_count} > 0.2)) {
+            printf("QUALITY: $exposure->{exp_id} has bad warp quality: %d / %d\n",
+                   $exposure->{warp_bad_quality} , $exposure->{warp_component_count});
+            $is_bad_quality = 1;
+        }
+        elsif ((defined($exposure->{diffRun_state}))&&($exposure->{diffRun_state} eq 'full')&&
+               ($exposure->{diff_bad_quality} / $exposure->{diff_component_count} > 0.5)) {
+            printf("QUALITY: $exposure->{exp_id} has bad diff quality: %d / %d\n",
+                    $exposure->{diff_bad_quality} , $exposure->{diff_component_count});
+            $is_bad_quality = 1;
+        }
+        # If we've detected a bad quality exposure, drop it, and tell the companion.
+        if ($is_bad_quality) {
+            unless ((defined($exposure->{diffRun_state}))&&
+                    ($exposure->{diffRun_state} eq 'full')) {
+                $needs_qstack = 1;
+                $needs_something_private = 1;
+                if ($companion) {
+                    $companion->{private} = 1;
+                    $companion->{pairwise} = 0;
+                    &update_this_exposure($companion);
+                }
+                $exposure->{private} = 1;
+                $exposure->{pairwise} = 0;
             }
-            $exposure->{private} = 1;
-            $exposure->{pairwise} = 0;
             $exposure->{data_state} = 'drop';
             &update_this_exposure($exposure);
 
         }
-#       if ($companion) { # Validate that there are no problems with the companion exposure
-#           if ($companion->{cam_quality}) { # Maybe other things here?
-#               $exposure->{private} = 1;
-#               $exposure->{data_state} = 'drop';
-#               &update_this_exposure($exposure);
-#               $needs_qstack = 1;
-#           }
-#       }
+        
         if  ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through.
             $can_qstack ++;
@@ -564 +621 @@
     }
 
+    my $warps = '';
+    foreach $exposure (@$exposures) {
+        if (($exposure->{data_state} != 'drop')&&
+            (S64_IS_NOT_NULL($exposure->{warp_id}))) {
+            $warps .= " -warp_id $exposure->{warp_id} ";
+        }
+    }
+
     my @utctime = gmtime();
     $utctime[5] += 1900;
@@ -570 +635 @@
     my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
     my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
-    my $data_group = "${label}.${date}";
+    my $data_group = "${label}.${proj_cell}.quick.${date}";
 
     my $command = "$stacktool ";
@@ -576 +641 @@
     $command .= " -dbname $dbname " if defined $dbname;
     $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
-    $command .= " -set_label ${label} -set_data_group ${proj_cell}.quick.${date} ";
+    $command .= " -set_label ${label} -set_data_group $data_group ";
     $command .= "  -set_workdir $workdir  -set_dist_group NODIST ";
     $command .= " -min_num 2 -set_reduction QUICKSTACK ";
+    $command .= " $warps ";
 
     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -589 +655 @@
     $command = "$stacktool ";
     $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -sassskyfile -data_group ${proj_cell}.quick.${date} ";
+    $command .= " -sassskyfile -data_group $data_group ";
     $command .= " -filter $filter -projection_cell ${proj_cell} ";
 
@@ -632 +698 @@
     unless (defined($label) && defined($filter) && defined($proj_cell)) {
         &my_die("Unable to perform stacktool. Insufficient information.", $lap_id);
+    }
+
+    my $warps = '';
+    foreach $exposure (@$exposures) {
+        if (($exposure->{data_state} != 'drop')&&
+            (S64_IS_NOT_NULL($exposure->{magicked}))&&
+            (S64_IS_NOT_NULL($exposure->{warp_id}))) {
+            $warps .= " -warp_id $exposure->{warp_id} ";
+        }
     }
 
@@ -640 +715 @@
     my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
     my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
-    my $data_group = "${label}.${date}";
+    my $data_group = "${label}.${proj_cell}.final.${date}";
 
     my $command = "$stacktool ";
@@ -646 +721 @@
     $command .= " -dbname $dbname " if defined $dbname;
     $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
-    $command .= " -set_label ${label} -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} ";
-    $command .= " -min_num 2 -set_reduction THREEPI_STACK ";
+    $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group ";
+    $command .= " -min_num 2 -set_reduction THREEPI_STACK -set_dist_group ${label} ";
+    $command .= " $warps ";
 
     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -659 +735 @@
     $command = "$stacktool ";
     $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -sassskyfile -data_group ${proj_cell}.final.${date} ";
+    $command .= " -sassskyfile -data_group $data_group ";
     $command .= " -filter $filter -projection_cell ${proj_cell} ";
 
@@ -734 +810 @@
             $command .= " -set_dist_group $exposure->{dist_group} ";
         }
-        
-        if ($exposure->{pairwise}) { # warpwarp
+        my $retry_command;
+        if (($exposure->{pairwise})&&(defined(${ $exposures }[$match_hash{$exposure->{chip_id}}]))) { # warpwarp
             my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}];
             $command .= " -definewarpwarp ";
-            $command .= "-input_label $label -template_label $label -backwards ";
+            $command .= "-input_label $label -template_label $label ";
             $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} ";
+            $retry_command = $command;
+            $command .= " -backwards "; # This usually works.
             $already_queued{$exposure->{warp_id}} = 1;
             $already_queued{$companion->{warp_id}} = 1;
@@ -763 +841 @@
         my $diff_id = $diff->{diff_id};
         unless (defined($diff_id)) {
-            $exposure->{data_state} = 'drop';
-            &update_this_exposure($exposure);
-        }
-        
+            if ($retry_command) {
+                ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $retry_command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("unable to perform difftool -definewarp(warp|stack): $error_code", $exposure->{lap_id}, $exposure->{proj_cell});
+                }
+                
+                $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata from difftool -definewarp(warp|stack)", $lap_id, "");
+                
+                $diff = ${ $diffs }[0];
+                $diff_id = $diff->{diff_id};
+            }
+            unless (defined($diff_id)) {
+                $exposure->{data_state} = 'drop';
+                &update_this_exposure($exposure);
+            }
+        }
     }
 }
@@ -872 +965 @@
     
     my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id ";
+    $command .= " -dbname $dbname " if defined $dbname;
     if (($exposure->{chip_id})&&(S64_IS_NOT_NULL($exposure->{chip_id}))) {
         $command .= " -set_chip_id  $exposure->{chip_id} ";

branches/eam_branches/ipp-20110505/ippScripts/scripts/receive_file.pl

@@ -224 +224 @@
         $stage = 'sky';
         # XXX: This should be skycell, but the distribution code uses exposure
-        $comp_name = 'exposure';
-        $current_component = $comp_name;
+        $comp_name = 'skycell_id';
+#        $current_component = $comp_name;
     } else {
         &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
@@ -237 +237 @@
         $new_workdir_value = "$workdir/$destdir";
     }
+
+    if ($stage eq 'sky') {
+        # the dbinfo file for a skyRun only has one component and it doesn't contain
+        # skycell_id which is the way components are listed in the dirinfo file.
+        my @ids = keys %$components;
+        &my_die( "unexpected number of components scalar @ids found in staticsky dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if scalar @ids != 1;
+        $current_component = $ids[0];
+    }
+
     my $component_dir;
     if ($current_component) {

branches/eam_branches/ipp-20110505/ippScripts/scripts/warp_skycell.pl

@@ -72 +72 @@
 
 my ($logDest, $traceDest);
+my $do_stats;
 if ($run_state eq 'new') {
     $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0);
     $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
+    $do_stats = 1;
 } elsif ($run_state eq 'update')  {
     $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 1);
@@ -162 +164 @@
 my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1);
 my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1);
-my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1);
+my $outputStats;
+if ($do_stats) {
+    $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1) if $do_stats;
+}
 my $configuration;
 
@@ -257 +262 @@
 # Run pswarp
 my $cmdflags;
-my $do_stats;
 unless ($no_op) {
     my $command = "$pswarp";
@@ -279 +283 @@
     if ($run_state eq 'new') {
         $command .= " -dumpconfig $configuration";
-        $do_stats = 1;
     } else {
         $command .= " -ipprc $configuration";