Index: /tags/ipp-ps2-20190404/ippScripts/scripts/nightly_science.pl =================================================================== --- /tags/ipp-ps2-20190404/ippScripts/scripts/nightly_science.pl (revision 40860) +++ /tags/ipp-ps2-20190404/ippScripts/scripts/nightly_science.pl (revision 40861) @@ -46,5 +46,5 @@ # Grab options -my ( $date, $datetime, $now, $camera, $dbname, $logfile, $verbose, $manual); +my ( $date, $datetime, $now, $camera, $dbname, $logfile, $verbose, $manual,$desdiffdt); my ( $help, $isburning, $force_stack_count, $force_diff_count, $force_registration, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode); my ( $registration_status, $burntool_status, $observing_status, $old_date); @@ -85,4 +85,5 @@ 'queue_dqstats' => \$queue_dqstats, 'clean_old' => \$clean_old, + 'desdiffdt=s' => \$desdiffdt, # TdB20190529delay time for making desperate iddfs on the fly ) or pod2usage ( 2 ); pod2usage( -msg => @@ -100,5 +101,6 @@ --this_filter_only Process only a single filter. --this_mode_only Process only a single clean mode. - --burntool_status Display Nexp Nimfile Nburntooled Nqueued for check_chips. + --burntool_status Display Nexp Nimfile Nburntooled Nqueued for check_chips. + --desdiffdt Default 3hrs. Modes: --check_stacks Confirm that stacks can be built. @@ -248,6 +250,6 @@ $datetime = DateTime->now(time_zone => 'UTC'); $datetime->set_year($year); + $datetime->set_day($day); $datetime->set_month($month); - $datetime->set_day($day); } } @@ -291,4 +293,14 @@ $metadata_out{nsRegistrationState} = 'REGISTERED'; } + +if (defined($desdiffdt) == 0) { + $desdiffdt = 1.5; +} + +#Build in a safety for desdiffdt +if ((defined($desdiffdt) == 1) && ($desdiffdt <= 0.5)) { + $desdiffdt = 0.5; +} + # # Mode selection @@ -1357,4 +1369,5 @@ my $cam_ref = $db->selectall_arrayref( $cam_sth ); my $warp_ref = $db->selectall_arrayref( $warp_sth ); + return($#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2); } @@ -1402,4 +1415,9 @@ diff_queue($date,$target,$filter,$pretend); + #Queue up desperate diffs if the conditions are right for them + if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { + desperate_diff_singles($date,$target,$filter,$pretend); + } + if (defined($pretend)) { add_to_macro_list('check_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter); @@ -1485,9 +1503,5 @@ foreach my $object_row (@{ $object_ref }) { - my $this_object = shift @{ $object_row }; -# my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; -# $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; -# $input_sth .= " ORDER BY dateobs "; - + my $this_object = shift @{ $object_row }; my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM "; $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; @@ -1620,6 +1634,5 @@ } - -sub desperate_diff_queue { +sub desperate_diff_singles { my $date = shift; my $target = shift; @@ -1627,4 +1640,7 @@ my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + my $this_date = undef; + my $chunk_name = undef; + my ($year,$month,$day,$hour,$min,$sec); my $db = init_gpc_db(); @@ -1637,4 +1653,255 @@ my $Npotential = 0; my $Nqueued = 0; + + #Before considering making diffs, we should check if there are no issues with summit download and/or registration. + #summitExp has limited information, so see if the number of exps in rawExp and summitExp match up before most recent dateobs of chunk + #be careful here, since dateobs is not quite completely the same in both tables. Therefore, allow numbers to be off by 1 + my $timenow=DateTime->now; + $timenow->subtract(minutes => 15); + $timenow->set_time_zone("UTC"); + + my $dateinit = $timenow->ymd; + my $raw_sth = "select count(dateobs),MAX(dateobs) FROM rawExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' "; + my ($nrawexps,$rawdate) = $db->selectrow_array( $raw_sth ); + + my $summit_sth = "select count(dateobs),MAX(dateobs) FROM summitExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' "; + my ($nsummitexps,$summitdate) = $db->selectrow_array( $summit_sth ); + + my $summittimediff = 9999; + if(($rawdate)&&($summitdate)) { + $rawdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/; + my $date1=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); + + $summitdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/; + my $date2=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); + + my $difference=$date2->delta_ms($date1); + + $summittimediff=(($difference->hours)*60.) + ($difference->minutes) + (($difference->seconds)/60.); + } + + #Run some basic checks + my $noff = abs($nsummitexps-$nrawexps); + if (($summittimediff > 10.)||($noff >10)) { + if ($debug == 1) { + print STDERR "desp_diff_singles: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n"; + } + return; + } + + foreach my $object_row (@{ $object_ref }) { + my $this_object = shift @{ $object_row }; + + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM "; + $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; + $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " ORDER BY dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + + # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. + my %comment_hash = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + my $this_quality = ${ $this_warp }[5]; + if ($this_quality != 0) { + print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n"; + } + else { + $comment_hash{$this_comment} = $this_exp_id; + } + } + # Exclude any warps that are not stored in the comment_hash. + my @keep_warps = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ((exists($comment_hash{$this_comment}))&& + ($comment_hash{$this_comment} == $this_exp_id)) { + push @keep_warps, $this_warp; + $this_date = ${ $this_warp }[2]; + $chunk_name = ${ $this_warp }[6]; + } + else { + print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; + } + } + @{ $warps } = @keep_warps; + + #find the time of the most recent exposure in that chunk + my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' "; + my $chunk = $db->selectall_arrayref( $input_chunk ); + foreach my $this_chunk (@{ $chunk }) { + $this_date = ${ $this_chunk }[0]; + } + + #compute and store some stats for potential on-the-fly desperate diffs + my $nwarps = ($#{ $warps } + 1); + my $timediff = 0; + my $now=DateTime->now; + $now->set_time_zone("UTC"); + + if($this_date) { + if($this_date=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/) { + my $dt=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); + my $difference=$now->delta_ms($dt); + $timediff=($difference->hours) + (($difference->minutes) + ($difference->seconds)/60.)/60.; + } + } + + #Consider the special case of on-the-fly desperate diffs + #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old + if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { + print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; + next; + } + if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { + next; + } + + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warps } = reverse @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } + + while ($#{ $warps } > -1) { + # The array is sorted in pairs of input/template. + my $template_warp = shift @{ $warps }; + my $input_warp = shift @{ $warps }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_singles: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } + } + $metadata_out{nsDiffPotential} += $Npotential; + $metadata_out{nsDiffQueued} += $Nqueued; +} + +sub desperate_diff_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + my $pretend = shift; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + my ($year,$month,$day,$hour,$min,$sec); + + my $db = init_gpc_db(); + + my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; + $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object"; + + my $object_ref = $db->selectall_arrayref( $obj_sth ); + + my $Npotential = 0; + my $Nqueued = 0; + + #Before considering making diffs, we should check if there are no issues with summit download and/or registration. + #summitExp has limited information, so see if the number of exps in rawExp and summitExp match up before most recent dateobs of chunk + #be careful here, since dateobs is not quite completely the same in both tables. Therefore, allow numbers to be off by 1 + my $timenow=DateTime->now; + $timenow->subtract(minutes => 15); + $timenow->set_time_zone("UTC"); + + my $dateinit = $timenow->ymd; + my $raw_sth = "select count(dateobs),MAX(dateobs) FROM rawExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' "; + my ($nrawexps,$rawdate) = $db->selectrow_array( $raw_sth ); + + my $summit_sth = "select count(dateobs),MAX(dateobs) FROM summitExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' "; + my ($nsummitexps,$summitdate) = $db->selectrow_array( $summit_sth ); + + my $summittimediff = 9999; + if(($rawdate)&&($summitdate)) { + $rawdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/; + my $date1=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); + + $summitdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/; + my $date2=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); + + my $difference=$date2->delta_ms($date1); + + $summittimediff=(($difference->hours)*60.) + ($difference->minutes) + (($difference->seconds)/60.); + } + + #Run some basic download/registration checks + my $noff = abs($nsummitexps-$nrawexps); + if (($summittimediff > 10.)||($noff >10)) { + if ($debug == 1) { + print STDERR "desp_diff_queue: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n"; + } + return; + } foreach my $object_row (@{ $object_ref }) {