Changeset 40855

Timestamp:

Aug 5, 2019, 3:35:37 PM (7 years ago)

Author:

tdeboer

Message:

on the fly desperate diffs added to nightly_science script

File:

• trunk/ippScripts/scripts/nightly_science.pl (modified) (11 diffs)

trunk/ippScripts/scripts/nightly_science.pl

@@ -46 +46 @@
 
 # Grab options
-my ( $date, $datetime, $now, $camera, $dbname, $logfile, $verbose, $manual);
+my ( $date, $datetime, $now, $camera, $dbname, $logfile, $verbose, $manual,$desdiffdt);
 my ( $help, $isburning, $force_stack_count, $force_diff_count, $force_registration, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
 my ( $registration_status, $burntool_status, $observing_status, $old_date);
@@ -85 +85 @@
     'queue_dqstats'        => \$queue_dqstats,
     'clean_old'            => \$clean_old,
+    'desdiffdt=s'             => \$desdiffdt, # TdB20190529delay time for making desperate iddfs on the fly
     ) or pod2usage ( 2 );
 pod2usage( -msg =>
@@ -100 +101 @@
            --this_filter_only     Process only a single filter.
            --this_mode_only       Process only a single clean mode.
-           --burntool_status       Display Nexp Nimfile Nburntooled Nqueued for check_chips.
+           --burntool_status      Display Nexp Nimfile Nburntooled Nqueued for check_chips.
+           --desdiffdt <hrs>      Default 3hrs.
         Modes:
            --check_stacks         Confirm that stacks can be built.
@@ -248 +250 @@
         $datetime = DateTime->now(time_zone => 'UTC');
         $datetime->set_year($year);
+        $datetime->set_day($day);
         $datetime->set_month($month);
-        $datetime->set_day($day);
     }
 }    
@@ -291 +293 @@
     $metadata_out{nsRegistrationState} = 'REGISTERED';
 }
+
+if (defined($desdiffdt) == 0) {
+        $desdiffdt = 1.5;
+}
+
+#Build in a safety for desdiffdt
+if ((defined($desdiffdt) == 1) && ($desdiffdt <= 0.5)) {
+        $desdiffdt = 0.5;
+}
+
 #
 # Mode selection
@@ -1357 +1369 @@
     my $cam_ref  = $db->selectall_arrayref( $cam_sth );
     my $warp_ref = $db->selectall_arrayref( $warp_sth );
+
     return($#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2);
 }
@@ -1402 +1415 @@
                 diff_queue($date,$target,$filter,$pretend);
 
+                #Queue up desperate diffs if the conditions are right for them  
+                if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) {
+                    desperate_diff_singles($date,$target,$filter,$pretend);
+                }
+
                 if (defined($pretend)) {
                     add_to_macro_list('check_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter);
@@ -1485 +1503 @@
     
     foreach my $object_row (@{ $object_ref }) {
-        my $this_object = shift @{ $object_row };
-#       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
-#       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
-#       $input_sth .= " ORDER BY dateobs ";
-        
+        my $this_object = shift @{ $object_row };       
         my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM ";
         $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
@@ -1620 +1634 @@
 }
 
-
-sub desperate_diff_queue {
+sub desperate_diff_singles {
     my $date = shift;
     my $target = shift;
@@ -1627 +1640 @@
     my $pretend = shift;
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+    my $this_date = undef;
+    my $chunk_name = undef;
+    my ($year,$month,$day,$hour,$min,$sec);
 
     my $db = init_gpc_db();
@@ -1637 +1653 @@
     my $Npotential = 0;
     my $Nqueued = 0;
+
+    #Before considering making diffs, we should check if there are no issues with summit download and/or registration. 
+    #summitExp has limited information, so see if the number of exps in rawExp and summitExp match up before most recent dateobs of chunk
+    #be careful here, since dateobs is not quite completely the same in both tables. Therefore, allow numbers to be off by 1
+    my $timenow=DateTime->now;
+    $timenow->subtract(minutes => 15);
+    $timenow->set_time_zone("UTC");
+
+    my $dateinit = $timenow->ymd;
+    my $raw_sth = "select count(dateobs),MAX(dateobs) FROM rawExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' ";
+    my ($nrawexps,$rawdate) = $db->selectrow_array( $raw_sth );
+
+    my $summit_sth = "select count(dateobs),MAX(dateobs) FROM summitExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' ";
+    my ($nsummitexps,$summitdate) = $db->selectrow_array( $summit_sth );
+ 
+    my $summittimediff = 9999;
+    if(($rawdate)&&($summitdate)) {
+        $rawdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
+        my $date1=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
+  
+        $summitdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
+        my $date2=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
+       
+        my $difference=$date2->delta_ms($date1);
+
+        $summittimediff=(($difference->hours)*60.) + ($difference->minutes) + (($difference->seconds)/60.);
+    }
+
+    #Run some basic checks
+    my $noff = abs($nsummitexps-$nrawexps);
+    if (($summittimediff > 10.)||($noff >10)) {
+        if ($debug == 1) {
+        print STDERR "desp_diff_singles: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n";
+        }
+        return;
+    }
+    
+    foreach my $object_row (@{ $object_ref }) {
+        my $this_object = shift @{ $object_row };
+        
+        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM ";
+        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
+        $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+        $input_sth .=   " ORDER BY dateobs ";
+
+        my $warps = $db->selectall_arrayref( $input_sth );
+
+        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+        my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            my $this_quality = ${ $this_warp }[5];
+            if ($this_quality != 0) {
+                print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
+            }
+            else {
+                $comment_hash{$this_comment} = $this_exp_id;
+            }
+        }
+        # Exclude any warps that are not stored in the comment_hash.
+        my @keep_warps = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            if ((exists($comment_hash{$this_comment}))&&
+                ($comment_hash{$this_comment} == $this_exp_id)) {
+                push @keep_warps, $this_warp;
+                $this_date  = ${ $this_warp }[2];
+                $chunk_name  = ${ $this_warp }[6];
+            }
+            else {
+                print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n";
+            }
+        }
+        @{ $warps } = @keep_warps;
+
+        #find the time of the most recent exposure in that chunk
+        my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' ";
+        my $chunk = $db->selectall_arrayref( $input_chunk );
+        foreach my $this_chunk (@{ $chunk }) {
+            $this_date = ${ $this_chunk }[0];
+        }
+ 
+        #compute and store some stats for potential on-the-fly desperate diffs
+        my $nwarps = ($#{ $warps } + 1);
+        my $timediff = 0;
+        my $now=DateTime->now;
+        $now->set_time_zone("UTC");
+
+        if($this_date) {
+            if($this_date=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/) {
+                my $dt=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
+                my $difference=$now->delta_ms($dt);
+                $timediff=($difference->hours) + (($difference->minutes) + ($difference->seconds)/60.)/60.;
+            }
+        }
+
+        #Consider the special case of on-the-fly desperate diffs
+        #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old 
+        if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) {
+            print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n";
+            next;
+        }
+        if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) {
+            next;
+        }
+
+        # We are attempting to do the missing diffs, so reverse the list of retained warps.
+        # Good objects with all visits will be skipped due to the duplicate check.
+        # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs.
+        @{ $warps } = reverse @keep_warps;
+
+        # Exclude the last entry if we do not have an even number of warps.
+        if (($#{ $warps } + 1) % 2 != 0) {
+            print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!";
+            my $rejected_warp = pop @{ $warps };
+            my $rejected_exp_id = ${ $rejected_warp }[0];
+            print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
+        }
+        
+        while ($#{ $warps } > -1) {
+            # The array is sorted in pairs of input/template.
+            my $template_warp = shift @{ $warps };
+            my $input_warp = shift @{ $warps };
+
+            my $input_exp_id = ${ $input_warp }[0];
+            my $input_comment = ${ $input_warp }[3];
+
+            my $template_exp_id = ${ $template_warp }[0];
+            my $template_comment = ${ $template_warp }[3];
+
+            my $input_warp_id = ${ $input_warp }[1];
+            my $template_warp_id = ${ $template_warp }[1];
+
+            my $input_warp_state = ${ $input_warp }[4];
+            my $template_warp_state = ${ $template_warp }[4];
+
+            my $input_warp_camQuality = ${ $input_warp }[5];
+            my $template_warp_camQuality = ${ $template_warp }[5];
+
+            $Npotential++;
+
+            unless (defined($input_warp_id) && defined($template_warp_id) &&
+                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
+                print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
+                if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
+                    # This should now never be reached.
+                    # CZW: Trigger backup plan here?  Or simply set up framework?
+                    print STDERR "  ...but this is due to a camera stage astrometry quality\n";
+                    $Npotential--;
+                }
+                next;
+            }
+
+            if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
+                $Nqueued++;
+                print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
+                next;
+            }
+            else {
+                print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
+            }
+            
+            my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
+            $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
+            $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+            $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures
+            $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
+            $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+            if (defined($reduction)) {
+                $cmd .= " -set_reduction $reduction ";
+            }
+        
+            if (defined($pretend)) {
+                $cmd .= ' -pretend ';
+            }
+            if ($debug == 1) {
+                $cmd .= ' -pretend ';
+                print STDERR "desp_diff_singles: $cmd\n";
+                print STDERR " $input_warp_id $template_warp_id\n";
+            }
+            
+            if (($debug == 0)&&(!defined($pretend))) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run ( command => $cmd, verbose => $verbose );
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                }
+                $Nqueued++;
+            }
+        }
+    }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+}
+
+sub desperate_diff_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+    my ($year,$month,$day,$hour,$min,$sec);
+
+    my $db = init_gpc_db();
+
+    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+
+    my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
+
+    #Before considering making diffs, we should check if there are no issues with summit download and/or registration. 
+    #summitExp has limited information, so see if the number of exps in rawExp and summitExp match up before most recent dateobs of chunk
+    #be careful here, since dateobs is not quite completely the same in both tables. Therefore, allow numbers to be off by 1
+    my $timenow=DateTime->now;
+    $timenow->subtract(minutes => 15);
+    $timenow->set_time_zone("UTC");
+
+    my $dateinit = $timenow->ymd;
+    my $raw_sth = "select count(dateobs),MAX(dateobs) FROM rawExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' ";
+    my ($nrawexps,$rawdate) = $db->selectrow_array( $raw_sth );
+
+    my $summit_sth = "select count(dateobs),MAX(dateobs) FROM summitExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' ";
+    my ($nsummitexps,$summitdate) = $db->selectrow_array( $summit_sth );
+ 
+    my $summittimediff = 9999;
+    if(($rawdate)&&($summitdate)) {
+        $rawdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
+        my $date1=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
+  
+        $summitdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
+        my $date2=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
+       
+        my $difference=$date2->delta_ms($date1);
+
+        $summittimediff=(($difference->hours)*60.) + ($difference->minutes) + (($difference->seconds)/60.);
+    }
+
+    #Run some basic download/registration checks
+    my $noff = abs($nsummitexps-$nrawexps);
+    if (($summittimediff > 10.)||($noff >10)) {
+        if ($debug == 1) {
+        print STDERR "desp_diff_queue: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n";
+        }
+        return;
+    }
     
     foreach my $object_row (@{ $object_ref }) {