Index: /trunk/ippScripts/scripts/nightly_science.pl =================================================================== --- /trunk/ippScripts/scripts/nightly_science.pl (revision 39827) +++ /trunk/ippScripts/scripts/nightly_science.pl (revision 39828) @@ -47,5 +47,5 @@ # Grab options my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual); -my ( $help, $isburning, $force_stack_count, $force_diff_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode); +my ( $help, $isburning, $force_stack_count, $force_diff_count, $force_registration, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode); my ( $registration_status, $burntool_status, $observing_status, $old_date); my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); @@ -65,4 +65,5 @@ 'force_stack_count' => \$force_stack_count, 'force_diff_count' => \$force_diff_count, + 'force_registration' => \$force_registration, 'this_target_only=s' => \$this_target_only, 'this_filter_only=s' => \$this_filter_only, @@ -285,4 +286,7 @@ $metadata_out{nsObservingState} = &get_observing_state($date); $metadata_out{nsRegistrationState} = &get_registration_state($date); +if ($force_registration) { + $metadata_out{nsRegistrationState} = 'REGISTERED'; +} # # Mode selection @@ -1435,14 +1439,12 @@ if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') { foreach my $target (sort (keys %science_config)) { - if ($science_config{$target}{DIFFABLE} == 1) { + # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now. + if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { foreach my $filter (@filter_list) { - multi_date_diff_queue($date,$target,$filter,$pretend); - } - } - if ((defined($science_config{$target}{OFFNIGHT_DIFFS})) && ($science_config{$target}{OFFNIGHT_DIFFS} == 1)) { - foreach my $filter (@filter_list) { - offnight_diff_queue($date,$target,$filter,$pretend); - } - } + # This one needs to return a state to see if we need to wait on stacking before checking again. + $metadata_out{nsDiffState} = desperate_diff_queue($date,$target,$filter,$pretend); + } + } + if ((defined($science_config{$target}{SELF_WSDIFFS})) && ($science_config{$target}{SELF_WSDIFFS} == 1)) { foreach my $filter (@filter_list) { @@ -1498,7 +1500,28 @@ my $this_comment = ${ $this_warp }[3]; my $this_exp_id = ${ $this_warp }[0]; - $comment_hash{$this_comment} = $this_exp_id; - } - + my $this_quality = ${ $this_warp }[5]; + if ($this_quality != 0) { + print STDERR "diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n"; + } + else { + $comment_hash{$this_comment} = $this_exp_id; + } + } + # Exclude any warps that are not stored in the comment_hash. + my @keep_warps = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ((exists($comment_hash{$this_comment}))&& + ($comment_hash{$this_comment} == $this_exp_id)) { + push @keep_warps, $this_warp; + } + else { + print STDERR "diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n"; + } + } + @{ $warps } = @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. if (($#{ $warps } + 1) % 2 != 0) { print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; @@ -1508,32 +1531,21 @@ } else { - print STDERR ": I should declare an exposure to be qualityy.\n"; - my @keep_warps = (); -# print "@{ $warps }\n"; - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - if ($comment_hash{$this_comment} == $this_exp_id) { - push @keep_warps, $this_warp; - } - else { - print STDERR "diff_queue: excluding $this_exp_id for $this_object\n"; - } - } - @{ $warps } = @keep_warps; -# print "@{ $warps }\n"; + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; } } while ($#{ $warps } > -1) { + # The array is sorted in pairs of input/template. my $input_warp = shift @{ $warps }; + my $template_warp = shift @{ $warps }; + my $input_exp_id = ${ $input_warp }[0]; my $input_comment = ${ $input_warp }[3]; - - my $template_warp = shift @{ $warps }; - my $template_exp_id = ${ $template_warp }[0]; - + my $template_comment = ${ $template_warp }[3]; + my $input_warp_id = ${ $input_warp }[1]; my $template_warp_id = ${ $template_warp }[1]; @@ -1551,4 +1563,6 @@ print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? print STDERR " ...but this is due to a camera stage astrometry quality\n"; $Npotential--; @@ -1559,6 +1573,9 @@ if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { $Nqueued++; - print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; + print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; } @@ -1601,4 +1618,160 @@ } + +sub desperate_diff_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + my $pretend = shift; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $db = init_gpc_db(); + + my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; + $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object"; + + my $object_ref = $db->selectall_arrayref( $obj_sth ); + + my $Npotential = 0; + my $Nqueued = 0; + + foreach my $object_row (@{ $object_ref }) { + my $this_object = shift @{ $object_row }; + + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM "; + $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; + $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " ORDER BY dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + + # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. + my %comment_hash = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + my $this_quality = ${ $this_warp }[5]; + if ($this_quality != 0) { + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n"; + } + else { + $comment_hash{$this_comment} = $this_exp_id; + } + } + # Exclude any warps that are not stored in the comment_hash. + my @keep_warps = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ((exists($comment_hash{$this_comment}))&& + ($comment_hash{$this_comment} == $this_exp_id)) { + push @keep_warps, $this_warp; + } + else { + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n"; + } + } + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warps } = reverse @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; + if ($#{ $warps} + 1 == 1) { + print STDERR ": I can do no diffs with only one exposure.\n"; + next; + } + else { + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } + } + + while ($#{ $warps } > -1) { + # In this mode, the array is sorted in pairs of template/input, counter to the standard queue. + my $template_warp = shift @{ $warps }; + my $input_warp = shift @{ $warps }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# $cmd .= " -pretend "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } + } + $metadata_out{nsDiffPotential} += $Npotential; + $metadata_out{nsDiffQueued} += $Nqueued; +# if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { +# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; +# } + +} + + sub multi_date_diff_queue { my $date = shift; @@ -1626,5 +1799,5 @@ my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM "; $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; - $input_sth .= " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " WHERE warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; $input_sth .= " ORDER BY dateobs "; Index: /trunk/ippconfig/recipes/nightly_science.config =================================================================== --- /trunk/ippconfig/recipes/nightly_science.config (revision 39827) +++ /trunk/ippconfig/recipes/nightly_science.config (revision 39828) @@ -324,4 +324,5 @@ REDUCTION STR SWEETSPOT ADDITIONAL_STACK_LABEL STR ecliptic.rp + DESPERATE_DIFFS BOOL TRUE # REDUCTION STR SSTF_4SIG END @@ -363,7 +364,7 @@ DIFF S16 5 END -TARGETS METADATA +TARGETS METADATA NAME STR NCU - DISTRIBUTION STR SweetSpot + DISTRIBUTION STR SweetSpot TESS STR RINGS.V3 OBSMODE STR NCU @@ -372,4 +373,6 @@ OFFNIGHT_DIFFS BOOL FALSE REDUCTION STR SWEETSPOT + DIST S16 300 + CHIP S16 300 WARP S16 300 DIFF S16 300 @@ -402,4 +405,5 @@ DIFFABLE BOOL TRUE OFFNIGHT_DIFFS BOOL FALSE + DESPERATE_DIFFS BOOL TRUE # CHIP S16 7 # DIFF S16 7