Changeset 29515 for branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl

Timestamp:

Oct 21, 2010, 2:45:13 PM (16 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl (modified) (11 diffs)

branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl

@@ -124 +124 @@
     defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or
     defined $test_mode or defined $clean_old or defined $check_mode or
-    defined $confirm_stacks;
+    defined $confirm_stacks or defined $burntool_stats;
 
 # Configurable parameters from our config file.
 my @target_list = ();
 my @filter_list = ();
+my %distribution_list = ();
 my %tessID_list = ();
 my %obsmode_list = ();
@@ -135 +136 @@
 my %cleanmods_list = ();
 my %stackable_list = ();
+my %extra_processing = ();
 my %reduction_class = ();
 my %macro_formats = ();
@@ -206 +208 @@
                 $this_target = ${ $tentry }{value};
                 push @target_list, $this_target;
+                $distribution_list{$this_target} = $this_target;
+            }
+            elsif (${ $tentry }{name} eq 'DISTRIBUTION') {
+                $distribution_list{$this_target} = ${ $tentry }{value};
             }
             elsif (${ $tentry }{name} eq 'TESS') {
@@ -222 +228 @@
                 $stackable_list{$this_target} = ${ $tentry }{value};
             }
+            elsif (${ $tentry }{name} eq 'EXTRA_PROCESSING') {
+                $extra_processing{$this_target} = ${ $tentry }{value};
+            }
             elsif (${ $tentry }{name} eq 'REDUCTION') {
                 $reduction_class{$this_target} = ${ $tentry }{value};
@@ -726 +735 @@
         my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target);
         if (defined($burntool_stats)) {
-            print "$Nexposures $Nimfiles $Nburntooled $Nalready\n";
+            print "BTSTATS: $date $target $Nexposures $Nimfiles $Nburntooled $Nalready\n";
         }
 
         if ($Nexposures == 0) {
             print STDERR "execute_chips: Target $target on $date had no exposures.\n";
-#           next;
+            next;
         }
         if ($Nalready != 0) {
             print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n";
-#            next;
+            next;
         }
         if ($Nimfiles != $Nburntooled) {
@@ -1183 +1192 @@
         &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     }
+    
+    
     return(0);
 }
@@ -1190 +1201 @@
     my $pretend = shift;
     foreach my $target (@target_list) {
-        if ($stackable_list{$target} == 1) {
-            foreach my $filter (@filter_list) {
+        foreach my $filter (@filter_list) {
+            if (exists($extra_processing{$target})) {
+                do_extra_processing($date,$target,$filter,$pretend);
+            }
+            if ($stackable_list{$target} == 1) {
                 my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
                 if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
@@ -1231 +1245 @@
                 }
             }
+            else {
+                # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
+            }
         }
-        else {
-            # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
-        }
-    }
-
+    }
 }
 
@@ -1291 +1304 @@
 
 #
+# Extra processing
+################################################################################
+
+sub do_extra_processing {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    if ($target eq 'OSS') {
+        my $db = init_gpc_db();
+
+        my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+        $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+        print STDERR "$obj_sth\n";
+        my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+        foreach my $object_row (@{ $object_ref }) {
+            my $this_object = shift @{ $object_row };
+            my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+            $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+            $input_sth .= " ORDER BY dateobs ";
+
+            my $warps = $db->selectall_arrayref( $input_sth );
+            
+            if (($#{ $warps } + 1) % 2 != 0) {
+                print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n";
+                die;
+            }
+            
+            while ($#{ $warps } > -1) {
+                my $input_warp = shift @{ $warps };
+                my $template_warp = shift @{ $warps };
+                my $input_exp_id = ${ $input_warp }[0];
+                my $template_exp_id = ${ $template_warp }[0];
+                
+                my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
+                $cmd .= "-input_label $label  -template_label $label ";
+                $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+                $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
+                $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+#               $cmd .= " -pretend ";
+                if (defined($pretend)) {
+                    $cmd .= ' -pretend ';
+                }
+                if ($debug == 1) {
+                    $cmd .= ' -pretend ';
+                }
+                print STDERR "EXTRA_PROCESSING: $cmd\n";
+                if (($debug == 0)&&(!defined($pretend))) {
+                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run ( command => $cmd, verbose => $verbose );
+                    unless ($success) {
+                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                        &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                    }
+                }
+            }
+        }
+    }
+}
+
+            
+
+
+#
 # Auto-Clean
 ################################################################################
@@ -1380 +1460 @@
     my $object   = $object_list{$target};
     my $comment = $comment_list{$target};
-    my $dist_group = $target;
+    my $dist_group = $distribution_list{$target};
     my $data_group = "${target}.${trunc_date}";
     my $tess_id = $tessID_list{$target};
@@ -1427 +1507 @@
         my $N = $metadata_out{N_MACROS};
         $metadata_out{"ns${N}Macro"} = $macro_formats{$proc_mode};
-        print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
+#       print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
         if (defined($date)&&(defined($target))) {