Changeset 30586 for branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl

Timestamp:

Feb 11, 2011, 4:29:48 PM (15 years ago)

Author:

watersc1

Message:

Trying to merge back into trunk.

Location:

branches/czw_branch/20101203

Files:

• . (modified) (1 prop)
• ippScripts/scripts/nightly_science.pl (modified) (21 diffs)

branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl

@@ -44 +44 @@
 my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual);
 my ( $help, $isburning, $force_stack_count, $force_diff_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
-my ( $registration_status, $burntool_status, $observing_status);
+my ( $registration_status, $burntool_status, $observing_status, $old_date);
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
 my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old);
@@ -66 +66 @@
     'registraion_status'   => \$registration_status,
     'burntool_status'      => \$burntool_status,
+    'old_date=s'           => \$old_date,
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
@@ -134 +135 @@
 unless ($success) {
     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $date, $PS_EXIT_SYS_ERROR);
 }
 
@@ -278 +279 @@
 
 $metadata_out{nsObservingState} = &get_observing_state($date);
+$metadata_out{nsRegistrationState} = &get_registration_state($date);
 #
 # Mode selection
@@ -396 +398 @@
     exit(0);
 }
-exit(10);
+
 if (defined($check_confirm_stacks) || defined($test_mode)) {
     $metadata_out{nsStackState} = 'CONFIRM_STACKING';
@@ -409 +411 @@
     exit(0);
 }
-
+exit(10);
 if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsSSState} = 'CHECKSWEETSPOT';
@@ -1097 +1099 @@
     my $pretend = shift;
 
+    my $Npotential = 0;
+    my $Nqueued = 0;
+    my $is_processing = 0;
+    my $is_registering;
+    if ($metadata_out{nsRegistrationState} eq 'REGISTERED') {
+        $is_registering = 0;
+    }
+    else {
+        $is_registering = 1;
+    }
+
     foreach my $target (sort (keys %science_config)) {
         if ($science_config{$target}{STACKABLE} == 1) {
@@ -1105 +1118 @@
                         print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
                     }
+                    $is_processing = 1;
                     $metadata_out{nsStackState} = 'FORCETOWARP';
                     next;
@@ -1120 +1134 @@
                     next;
                 }
+                $Npotential++;
                 if ($Nalready != 0) {
+                    $Nqueued++;
                     if ($debug == 1) {
                         print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
@@ -1137 +1153 @@
                     }
                     stack_queue($date,$target,$filter);
+                    $Nqueued ++;
                 }
                 if (defined($pretend)) {
@@ -1152 +1169 @@
         }
     }
+    $metadata_out{nsStackPotential} = $Npotential;
+    $metadata_out{nsStackQueued}    = $Nqueued;
+    if (($Npotential == $Nqueued)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)&&($is_registering == 0)) {
+        $metadata_out{nsStackState} = 'FINISHED_STACKS';
+    }   
 }
 
@@ -1231 +1253 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
 
-    my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
+    my $count = 0;
+    
+    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
     my $data_ref = $db->selectall_arrayref( $sth );
-
-    return($#{ $data_ref } + 1);
+    $count += $#{ $data_ref } + 1;
+
+    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
+    $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    return($count);
 }
 
@@ -1272 +1301 @@
     my $pretend = shift;
 
+    my $Npotential = 0;
+    my $Nnoexp     = 0;
+    my $is_processing = 0;
+    my $is_registering;
+    if ($metadata_out{nsRegistrationState} eq 'REGISTERED') {
+        $is_registering = 0;
+    }
+    else {
+        $is_registering = 1;
+    }
+
     foreach my $target (sort (keys %science_config)) {
         if ($science_config{$target}{DIFFABLE} == 1) {
             foreach my $filter (@filter_list) {
+                $Npotential ++;
                 my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter);
                 if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
@@ -1280 +1321 @@
                         print STDERR "execute_diffs: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
                     }
-                    $metadata_out{nsDiffState} = 'FORCETOWARP';
-                    next;
-                }
+                    $is_processing = 1;
+                }
                 if ($NprocChips == 0) {
+                    $Nnoexp ++;
                     if ($debug == 1) {
                         print STDERR "execute_diffs: Target $target in filter $filter on $date has no exposures.\n";
@@ -1311 +1352 @@
         }
     }
+    if ($debug == 1) {
+        print "$metadata_out{nsObservingState} $metadata_out{nsDiffPotential} $metadata_out{nsDiffQueued} $is_processing $is_registering\n";
+    }
+
+    if ($metadata_out{nsObservingState} eq 'END_OF_NIGHT') {
+        if ($is_processing == 1) {
+            $metadata_out{nsDiffState} = 'DIFFING';
+        }
+        elsif ($is_registering == 0) {
+            if ($Npotential == $Nnoexp) {
+                $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+            }
+            elsif ($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued}) {
+                $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+            }
+        }
+        else {
+            $metadata_out{nsDiffState} = 'DIFFING';
+        }
+    }
+    else {
+        $metadata_out{nsDiffState} = 'DIFFING';
+    }
+
+#     if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) {
+#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#     } 
+
+#     if ($is_processing == 1) {
+#       $metadata_out{nsDiffState} = 'DIFFING';
+#     }
 }
 
@@ -1326 +1398 @@
 
     my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
     
     foreach my $object_row (@{ $object_ref }) {
         my $this_object = shift @{ $object_row };
-        my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
-        $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
-        $input_sth .= " ORDER BY dateobs ";
+#       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+#       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+#       $input_sth .= " ORDER BY dateobs ";
         
+        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
+        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
+        $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+        $input_sth .=   " ORDER BY dateobs ";
+
         my $warps = $db->selectall_arrayref( $input_sth );
+
+        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+        my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            $comment_hash{$this_comment} = $this_exp_id;
+        }
         
         if (($#{ $warps } + 1) % 2 != 0) {
-            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps }\n";
-            next;
+            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
+            if ($#{ $warps} + 1 == 1) {
+                print STDERR ": I can do no diffs with only one exposure.\n";
+                next;
+            }
+            else {
+                print STDERR ": I should declare an exposure to be faulty.\n";
+                my @keep_warps = ();
+#               print "@{ $warps }\n";
+                foreach my $this_warp (@{ $warps }) {
+                    my $this_comment = ${ $this_warp }[3];
+                    my $this_exp_id  = ${ $this_warp }[0];
+                    if ($comment_hash{$this_comment} == $this_exp_id) {
+                        push @keep_warps, $this_warp;
+                    }
+                    else {
+                        print STDERR "diff_queue: excluding $this_exp_id for $this_object\n";
+                    }
+                }
+                @{ $warps } = @keep_warps;
+#               print "@{ $warps }\n";
+            }
         }
         
         while ($#{ $warps } > -1) {
             my $input_warp = shift @{ $warps };
+            my $input_exp_id = ${ $input_warp }[0];
+            my $input_comment = ${ $input_warp }[3];
+
+            
             my $template_warp = shift @{ $warps };
-            my $input_exp_id = ${ $input_warp }[0];
+
             my $template_exp_id = ${ $template_warp }[0];
             
             my $input_warp_id = ${ $input_warp }[1];
             my $template_warp_id = ${ $template_warp }[1];
+
+            my $input_warp_state = ${ $input_warp }[4];
+            my $template_warp_state = ${ $template_warp }[4];
             
+            $Npotential++;
+
+            unless (defined($input_warp_id) && defined($template_warp_id) &&
+                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
+                print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed\n";
+                next;
+            }
+
             if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
-                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n";
+                $Nqueued++;
+                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
                 next;
             }
@@ -1366 +1490 @@
                 $cmd .= ' -pretend ';
                 print STDERR "diff_queue: $cmd\n";
+                print STDERR " $input_warp_id $template_warp_id\n";
             }
             
@@ -1375 +1500 @@
                     &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
                 }
-            }
-        }
-    }
+                $Nqueued++;
+            }
+        }
+    }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#      }        
+
 }
             
@@ -1493 +1625 @@
         }
     }   
-
+#    print "$datetime $eon_dt\n";
     if (DateTime->compare($datetime,$eon_dt) < 1) {
         return("OBSERVING");
@@ -1501 +1633 @@
     }
 }
+
+# This basically does the end of night check, but does it "the hard way," to prevent the time from fooling us.
+sub get_registration_state {
+    my $date = shift;
+
+    foreach my $eon (keys %eon_config) {
+        my $command = "$regtool -processedexp -simple ";
+        $command .= " -dbname $dbname ";
+        $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 ";
+        $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT});
+        $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE});
+        $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE});
+        $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT});
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run ( command => $command, verbose => $verbose );
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+        }
+        my @eon_exposures = split /\n/, (join '', @$stdout_buf);
+        if ($#eon_exposures >= 0) {
+            return("REGISTERED");
+        }
+    }   
+    return("NOT_REGISTERED");
+}
+
+    
 
 sub get_tool_parameters {
@@ -1620 +1781 @@
     my $stage = shift; # stage name
     my $stage_id = shift; #  identifier
+    my $date = shift;
     my $exit_code = shift; # Exit code
-    # outputImage and path_base are globals
 
     carp($msg);