Changeset 39830

Timestamp:

Nov 29, 2016, 12:34:26 PM (10 years ago)

Author:

watersc1

Message:

Merge from trunk/branch of updated nightly_science code with improved diff queue handling.

Location:

tags/ipp-20141024

Files:

• ippScripts/scripts/nightly_science.pl (modified) (10 diffs, 1 prop)
• ippconfig/recipes/nightly_science.config (modified) (8 diffs, 1 prop)

tags/ipp-20141024/ippScripts/scripts/nightly_science.pl

@@ -47 +47 @@
 # Grab options
 my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual);
-my ( $help, $isburning, $force_stack_count, $force_diff_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
+my ( $help, $isburning, $force_stack_count, $force_diff_count, $force_registration, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
 my ( $registration_status, $burntool_status, $observing_status, $old_date);
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
@@ -65 +65 @@
     'force_stack_count'    => \$force_stack_count,
     'force_diff_count'     => \$force_diff_count,
+    'force_registration'   => \$force_registration,
     'this_target_only=s'   => \$this_target_only,
     'this_filter_only=s'   => \$this_filter_only,
@@ -285 +286 @@
 $metadata_out{nsObservingState} = &get_observing_state($date);
 $metadata_out{nsRegistrationState} = &get_registration_state($date);
+if ($force_registration) {
+    $metadata_out{nsRegistrationState} = 'REGISTERED';
+}
 #
 # Mode selection
@@ -1435 +1439 @@
     if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') {
         foreach my $target (sort (keys %science_config)) {
-            if ($science_config{$target}{DIFFABLE} == 1) {
+            # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now.
+            if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) {
                 foreach my $filter (@filter_list) {
-                    multi_date_diff_queue($date,$target,$filter,$pretend);
-                }
-            }
-            if ((defined($science_config{$target}{OFFNIGHT_DIFFS})) && ($science_config{$target}{OFFNIGHT_DIFFS} == 1)) {
-                foreach my $filter (@filter_list) {
-                    offnight_diff_queue($date,$target,$filter,$pretend);
-                }
-            }
+                    # This one needs to return a state to see if we need to wait on stacking before checking again.
+                    $metadata_out{nsDiffState} = desperate_diff_queue($date,$target,$filter,$pretend);
+                }
+            }
+
             if ((defined($science_config{$target}{SELF_WSDIFFS})) && ($science_config{$target}{SELF_WSDIFFS} == 1)) {
                 foreach my $filter (@filter_list) {
@@ -1498 +1500 @@
             my $this_comment = ${ $this_warp }[3];
             my $this_exp_id  = ${ $this_warp }[0];
-            $comment_hash{$this_comment} = $this_exp_id;
-        }
-        
+            my $this_quality = ${ $this_warp }[5];
+            if ($this_quality != 0) {
+                print STDERR "diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
+            }
+            else {
+                $comment_hash{$this_comment} = $this_exp_id;
+            }
+        }
+        # Exclude any warps that are not stored in the comment_hash.
+        my @keep_warps = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            if ((exists($comment_hash{$this_comment}))&&
+                ($comment_hash{$this_comment} == $this_exp_id)) {
+                push @keep_warps, $this_warp;
+            }
+            else {
+                print STDERR "diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n";
+            }
+        }
+        @{ $warps } = @keep_warps;
+
+        # Exclude the last entry if we do not have an even number of warps.
         if (($#{ $warps } + 1) % 2 != 0) {
             print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
@@ -1508 +1531 @@
             }
             else {
-                print STDERR ": I should declare an exposure to be qualityy.\n";
-                my @keep_warps = ();
-#               print "@{ $warps }\n";
-                foreach my $this_warp (@{ $warps }) {
-                    my $this_comment = ${ $this_warp }[3];
-                    my $this_exp_id  = ${ $this_warp }[0];
-                    if ($comment_hash{$this_comment} == $this_exp_id) {
-                        push @keep_warps, $this_warp;
-                    }
-                    else {
-                        print STDERR "diff_queue: excluding $this_exp_id for $this_object\n";
-                    }
-                }
-                @{ $warps } = @keep_warps;
-#               print "@{ $warps }\n";
+                my $rejected_warp = pop @{ $warps };
+                my $rejected_exp_id = ${ $rejected_warp }[0];
+                print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
             }
         }
         
         while ($#{ $warps } > -1) {
+            # The array is sorted in pairs of input/template.
             my $input_warp = shift @{ $warps };
+            my $template_warp = shift @{ $warps };
+
             my $input_exp_id = ${ $input_warp }[0];
             my $input_comment = ${ $input_warp }[3];
 
-            
-            my $template_warp = shift @{ $warps };
-
             my $template_exp_id = ${ $template_warp }[0];
-            
+            my $template_comment = ${ $template_warp }[3];
+
             my $input_warp_id = ${ $input_warp }[1];
             my $template_warp_id = ${ $template_warp }[1];
@@ -1551 +1563 @@
                 print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
                 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
+                    # This should now never be reached.
+                    # CZW: Trigger backup plan here?  Or simply set up framework?
                     print STDERR "  ...but this is due to a camera stage astrometry quality\n";
                     $Npotential--;
@@ -1559 +1573 @@
             if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
                 $Nqueued++;
-                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
+                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
                 next;
+            }
+            else {
+                print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
             }
             
@@ -1601 +1618 @@
 }
 
+
+sub desperate_diff_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $db = init_gpc_db();
+
+    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+
+    my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
+    
+    foreach my $object_row (@{ $object_ref }) {
+        my $this_object = shift @{ $object_row };
+        
+        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM ";
+        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
+        $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+        $input_sth .=   " ORDER BY dateobs ";
+
+        my $warps = $db->selectall_arrayref( $input_sth );
+
+        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+        my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            my $this_quality = ${ $this_warp }[5];
+            if ($this_quality != 0) { 
+                print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
+            }
+            else {
+                $comment_hash{$this_comment} = $this_exp_id;
+            }
+        }
+        # Exclude any warps that are not stored in the comment_hash.
+        my @keep_warps = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            if ((exists($comment_hash{$this_comment}))&&
+                ($comment_hash{$this_comment} == $this_exp_id)) {
+                push @keep_warps, $this_warp;
+            }
+            else {
+                print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n";
+            }
+        }
+        # We are attempting to do the missing diffs, so reverse the list of retained warps.
+        # Good objects with all visits will be skipped due to the duplicate check.
+        # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs.
+        @{ $warps } = reverse @keep_warps;
+                    
+        # Exclude the last entry if we do not have an even number of warps.
+        if (($#{ $warps } + 1) % 2 != 0) {
+            print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
+            if ($#{ $warps} + 1 == 1) {
+                print STDERR ": I can do no diffs with only one exposure.\n";
+                next;
+            }
+            else {
+                my $rejected_warp = pop @{ $warps };
+                my $rejected_exp_id = ${ $rejected_warp }[0];
+                print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
+            }
+        }
+        
+        while ($#{ $warps } > -1) {
+            # In this mode, the array is sorted in pairs of template/input, counter to the standard queue.
+            my $template_warp = shift @{ $warps };
+            my $input_warp = shift @{ $warps };
+
+            my $input_exp_id = ${ $input_warp }[0];
+            my $input_comment = ${ $input_warp }[3];
+
+            my $template_exp_id = ${ $template_warp }[0];
+            my $template_comment = ${ $template_warp }[3];
+
+            my $input_warp_id = ${ $input_warp }[1];
+            my $template_warp_id = ${ $template_warp }[1];
+
+            my $input_warp_state = ${ $input_warp }[4];
+            my $template_warp_state = ${ $template_warp }[4];
+
+            my $input_warp_camQuality = ${ $input_warp }[5];
+            my $template_warp_camQuality = ${ $template_warp }[5];
+
+            $Npotential++;
+
+            unless (defined($input_warp_id) && defined($template_warp_id) &&
+                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
+                print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
+                if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
+                    # This should now never be reached.
+                    # CZW: Trigger backup plan here?  Or simply set up framework?
+                    print STDERR "  ...but this is due to a camera stage astrometry quality\n";
+                    $Npotential--;
+                }
+                next;
+            }
+
+            if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
+                $Nqueued++;
+                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
+                next;
+            }
+            else {
+                print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
+            }
+            
+            my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
+            $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
+            $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+            $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures
+            $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
+            $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+#               $cmd .= " -pretend ";
+            if (defined($reduction)) {
+                $cmd .= " -set_reduction $reduction ";
+            }
+
+            if (defined($pretend)) {
+                $cmd .= ' -pretend ';
+            }
+            if ($debug == 1) {
+                $cmd .= ' -pretend ';
+                print STDERR "desp_diff_queue: $cmd\n";
+                print STDERR " $input_warp_id $template_warp_id\n";
+            }
+            
+            if (($debug == 0)&&(!defined($pretend))) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run ( command => $cmd, verbose => $verbose );
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                }
+                $Nqueued++;
+            }
+        }
+    }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#      }        
+
+}
+
+
 sub multi_date_diff_queue {
     my $date = shift;
@@ -1626 +1799 @@
         my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
         $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
-        $input_sth .=   " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+        $input_sth .=   " WHERE warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
         $input_sth .=   " ORDER BY dateobs ";
 

tags/ipp-20141024/ippconfig/recipes/nightly_science.config

@@ -8 +8 @@
   MODE           STR WARP
   COMMAND        STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@
-#  COMMAND        STR true
-  RETENTION_TIME U16 2
+  RETENTION_TIME U16 3
 END
 CLEAN_MODES METADATA
@@ -69 +68 @@
 
 TARGETS MULTI
+# TARGETS METADATA
+#   NAME         STR  LIGO
+#   DISTRIBUTION STR  LIGO
+#   TESS         STR  RINGS.V3
+#   OBSMODE      STR  LIGO
+#   STACKABLE      BOOL FALSE
+#   DIFFABLE       BOOL FALSE
+#   OFFNIGHT_DIFFS BOOL FALSE
+#   DIST           S16  15
+#   CHIP           S16  5
+#   WARP           S16  5
+#   DIFF           S16  5  
+# END
 TARGETS METADATA
   NAME      STR  M31
@@ -87 +99 @@
 END
 TARGETS METADATA
+  NAME      STR CFA
+  DISTRIBUTION STR CFA
+  TESS      STR MD07.V3
+  OBSMODE   STR CFA
+#  OBJECT    STR MD07%
+  STACKABLE BOOL TRUE
+  MIN_STACK U16 8
+  DIFFABLE  BOOL FALSE
+  OFFNIGHT_DIFFS BOOL FALSE
+  REDUCTION STR MD_DEFAULT
+#  DIST S16 30  
+END
+TARGETS METADATA
   NAME         STR SNIa
   DISTRIBUTION STR SNIa.test
@@ -94 +119 @@
   DIFFABLE    BOOL FALSE
   OFFNIGHT_DIFFS BOOL FALSE
+#  REDUCTION STR SNIA_DEFAULT
   ## change days till cleanup
-  DIST      S16 300
-#  CHIP           S16  7
-  WARP      S16 300 
-  DIFF      S16 300 
+  DIST      S16 20
+  CHIP      S16  2
+  WARP      S16  2 
+  DIFF      S16  2 
+END
+TARGETS METADATA
+  NAME         STR  QUB
+  DISTRIBUTION STR  NODIST 
+  TESS         STR  RINGS.V3
+  OBSMODE      STR  QUB
+  STACKABLE      BOOL FALSE
+  DIFFABLE       BOOL FALSE
+  OFFNIGHT_DIFFS BOOL FALSE
+  REDUCTION STR QUB_DEFAULT
+  DIST           S16  15
+  CHIP           S16  5 
+  WARP           S16  300
+  DIFF           S16  300
 END
 TARGETS METADATA
@@ -280 +320 @@
   STACKABLE   BOOL FALSE
   DIFFABLE    BOOL TRUE
+  OFFNIGHT_DIFFS BOOL FALSE
+  REDUCTION STR SWEETSPOT
+  ADDITIONAL_STACK_LABEL STR ecliptic.rp
+  DESPERATE_DIFFS BOOL TRUE
+#  REDUCTION STR SSTF_4SIG
+END
+TARGETS METADATA
+  NAME         STR ESS
+  DISTRIBUTION STR SweetSpot
+  TESS         STR RINGS.V3
+  OBSMODE      STR ESS
+  STACKABLE   BOOL FALSE
+  DIFFABLE    BOOL TRUE
+  OFFNIGHT_DIFFS BOOL FALSE
   REDUCTION STR SWEETSPOT
   ADDITIONAL_STACK_LABEL STR ecliptic.rp
@@ -285 +339 @@
 END
 TARGETS METADATA
-  NAME         STR ESS
+  NAME         STR MSS
   DISTRIBUTION STR SweetSpot
   TESS         STR RINGS.V3
-  OBSMODE      STR ESS
+  OBSMODE      STR MSS
   STACKABLE   BOOL FALSE
   DIFFABLE    BOOL TRUE
+  OFFNIGHT_DIFFS BOOL FALSE
   REDUCTION STR SWEETSPOT
   ADDITIONAL_STACK_LABEL STR ecliptic.rp
@@ -296 +351 @@
 END
 TARGETS METADATA
-  NAME         STR MSS
+  NAME         STR FSS
   DISTRIBUTION STR SweetSpot
   TESS         STR RINGS.V3
-  OBSMODE      STR MSS
-  STACKABLE   BOOL FALSE
-  DIFFABLE    BOOL TRUE
+  OBSMODE      STR FSS
+  STACKABLE   BOOL FALSE
+  DIFFABLE    BOOL FALSE
+  OFFNIGHT_DIFFS BOOL FALSE
   REDUCTION STR SWEETSPOT
-  ADDITIONAL_STACK_LABEL STR ecliptic.rp
-#  REDUCTION STR SSTF_4SIG
+  CHIP      S16 5
+  WARP      S16 300
+  DIFF      S16 300
+END
+TARGETS  METADATA
+  NAME          STR NCU
+  DISTRIBUTION  STR SweetSpot
+  TESS          STR RINGS.V3
+  OBSMODE       STR NCU
+  STACKABLE     BOOL FALSE
+  DIFFABLE      BOOL FALSE
+  OFFNIGHT_DIFFS BOOL FALSE
+  REDUCTION     STR SWEETSPOT
+  DIST          S16 300
+  CHIP          S16 300
+  WARP          S16 300
+  DIFF          S16 300
 END
 TARGETS METADATA
@@ -332 +403 @@
   STACKABLE   BOOL FALSE
   DIFFABLE    BOOL TRUE
-  OFFNIGHT_DIFFS BOOL TRUE
-  CHIP           S16  7
-  DIFF           S16  7
+  OFFNIGHT_DIFFS BOOL FALSE 
+  DESPERATE_DIFFS BOOL TRUE
+#  CHIP          S16  7
+#  DIFF          S16  7
 #  CHIP         S16 10
 #  DIFF         S16 12