Index: trunk/ippScripts/scripts/nightly_science.pl =================================================================== --- trunk/ippScripts/scripts/nightly_science.pl (revision 41994) +++ trunk/ippScripts/scripts/nightly_science.pl (revision 42154) @@ -142,4 +142,7 @@ my %unique_filter_hash = (); +#set up the camera config +my $ipprcam = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up camera", $PS_EXIT_CONFIG_ERROR ); # IPP camera configuration + # Grab the configuration data. my $conf_cmd = "$ppConfigDump -camera $camera -dump-recipe NIGHTLY_SCIENCE -"; @@ -1391,4 +1394,29 @@ foreach my $target (sort (keys %science_config)) { if ($science_config{$target}{DIFFABLE} == 1) { + + my $config_reduction = "DEFAULT"; + if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};} + print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n"; + + my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use + &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; + + my $recipeData; + { + # Get the PSASTRO recipe + my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR); + } + $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR); + } + + my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); + print STDERR "execute_diffs: Max FWHM= $maxFWHM for $target\n"; + foreach my $filter (@filter_list) { $Npotential ++; @@ -1413,9 +1441,9 @@ print STDERR "execute_diffs: Target $target in filter $filter on $date has exposures and will be queued.\n"; } - diff_queue($date,$target,$filter,$pretend); + diff_queue($date,$target,$filter,$maxFWHM,$pretend); #Queue up desperate diffs if the conditions are right for them if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { - desperate_diff_singles($date,$target,$filter,$pretend); + desperate_diff_singles($date,$target,$filter,$maxFWHM,$pretend); } @@ -1459,8 +1487,32 @@ if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') { foreach my $target (sort (keys %science_config)) { + + my $config_reduction = "DEFAULT"; + if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};} + print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n"; + + my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use + &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; + + my $recipeData; + { + # Get the PSASTRO recipe + my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR); + } + $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR); + } + + my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); + # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now. if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { foreach my $filter (@filter_list) { - desperate_diff_queue($date,$target,$filter,$pretend); + desperate_diff_queue($date,$target,$filter,$maxFWHM,$pretend); } } @@ -1488,4 +1540,5 @@ my $target = shift; my $filter = shift; + my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -1507,4 +1560,5 @@ $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " AND camProcessedExp.fwhm_major <= $maxFWHM "; $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs "; @@ -1639,4 +1693,5 @@ my $target = shift; my $filter = shift; + my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -1694,5 +1749,5 @@ my $this_chunk = shift @{ $object_row }; - my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM "; + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)),camProcessedExp.fwhm_major FROM "; $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; @@ -1700,7 +1755,9 @@ my $warps = $db->selectall_arrayref( $input_sth ); + my $warpsBQ = $db->selectall_arrayref( $input_sth );; # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. my %comment_hash = (); + my %comment_hash_good = (); foreach my $this_warp (@{ $warps }) { my $this_comment = ${ $this_warp }[3]; @@ -1708,18 +1765,25 @@ my $this_quality = ${ $this_warp }[5]; my $this_state = ${ $this_warp }[4]; - if (($this_quality != 0) || ($this_state eq 'drop') ) { - print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; + my $this_fwhm = ${ $this_warp }[7]; + if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { + print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; } else { + $comment_hash_good{$this_comment} = $this_exp_id; + } + + #also save the entries which are ok, but not best quality + if (($this_quality == 0) && ($this_state ne 'drop') ) { $comment_hash{$this_comment} = $this_exp_id; } } - # Exclude any warps that are not stored in the comment_hash. + + # Exclude any warps that are not stored in the comment_hash_good. my @keep_warps = (); foreach my $this_warp (@{ $warps }) { my $this_comment = ${ $this_warp }[3]; my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash{$this_comment}))&& - ($comment_hash{$this_comment} == $this_exp_id)) { + if ((exists($comment_hash_good{$this_comment}))&& + ($comment_hash_good{$this_comment} == $this_exp_id)) { push @keep_warps, $this_warp; $this_date = ${ $this_warp }[2]; @@ -1731,5 +1795,6 @@ } @{ $warps } = @keep_warps; - + my $nwarps = ($#{ $warps } + 1); + #find the time of the most recent exposure in that chunk my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' "; @@ -1740,5 +1805,4 @@ #compute and store some stats for potential on-the-fly desperate diffs - my $nwarps = ($#{ $warps } + 1); my $timediff = 0; my $now=DateTime->now; @@ -1752,99 +1816,243 @@ } } - - #Consider the special case of on-the-fly desperate diffs - #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old - if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { - print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; - next; - } - if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { - next; - } - - # We are attempting to do the missing diffs, so reverse the list of retained warps. - # Good objects with all visits will be skipped due to the duplicate check. - # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. - @{ $warps } = reverse @keep_warps; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warps } + 1) % 2 != 0) { - print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; - my $rejected_warp = pop @{ $warps }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; - } + + ################################ + #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs + #Conversely, less than 3 good quality warps might result in dirty diffs + ################################ + if($nwarps >= 3) { + #Consider the special case of on-the-fly desperate diffs + #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old + if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { + print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; + next; + } + if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { + next; + } + + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warps } = reverse @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } - while ($#{ $warps } > -1) { - # The array is sorted in pairs of input/template. - my $template_warp = shift @{ $warps }; - my $input_warp = shift @{ $warps }; + while ($#{ $warps } > -1) { + # The array is sorted in pairs of input/template. + my $template_warp = shift @{ $warps }; + my $input_warp = shift @{ $warps }; - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && - ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_singles: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } + + } else { + #Consider the case of on-the-fly dirty desperate diffs + #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old + + # Exclude any warps that are not stored in the more general comment_hash. + my @keep_warps = (); + foreach my $this_warp (@{ $warpsBQ }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ((exists($comment_hash{$this_comment}))&& + ($comment_hash{$this_comment} == $this_exp_id)) { + push @keep_warps, $this_warp; + $this_date = ${ $this_warp }[2]; + $chunk_name = ${ $this_warp }[6]; + } + else { + print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; + } + } + @{ $warpsBQ } = @keep_warps; + + if ($nwarps >= 2) { + #one regular diff has already been made. So, need only one more. Strip a good entry from the array + my $nrem = 0; + my @keep_warps = (); + foreach my $this_warp (@{ $warpsBQ }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + my $this_fwhm = ${ $this_warp }[7]; + if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { + $nrem++; + print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; + } else { + push @keep_warps, $this_warp; + } } - next; + @{ $warpsBQ } = @keep_warps; } - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warpsBQ } = reverse @{ $warpsBQ }; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warpsBQ } + 1) % 2 != 0) { + print STDERR "desp_diff_singles: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; + my $rejected_warp = pop @{ $warpsBQ }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; } - else { - print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + my $nwarpsBQ = ($#{$warpsBQ} + 1); + + + if (($nwarpsBQ >= 2) && ($timediff < $desdiffdt)) { + print STDERR "desp_diff_singles: There are potential dirty desperate diffs to be done, but the time criterium is not met.\n"; + next; + } + if (($nwarpsBQ <= 1) || ($timediff < $desdiffdt)) { + next; + } + + while ($#{ $warpsBQ } > -1) { + # The array is sorted in pairs of input/template. + my $template_warp = shift @{ $warpsBQ }; + my $input_warp = shift @{ $warpsBQ }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_singles: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } } - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "desp_diff_singles: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } + } } @@ -1858,4 +2066,5 @@ my $target = shift; my $filter = shift; + my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -1911,5 +2120,5 @@ my $this_chunk = shift @{ $object_row }; - my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM "; + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,camProcessedExp.fwhm_major FROM "; $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; @@ -1917,7 +2126,9 @@ my $warps = $db->selectall_arrayref( $input_sth ); + my $warpsBQ = $db->selectall_arrayref( $input_sth );; # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. my %comment_hash = (); + my %comment_hash_good = (); foreach my $this_warp (@{ $warps }) { my $this_comment = ${ $this_warp }[3]; @@ -1925,115 +2136,262 @@ my $this_quality = ${ $this_warp }[5]; my $this_state = ${ $this_warp }[4]; - if (($this_quality != 0) || ($this_state eq 'drop') ) { - print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; + my $this_fwhm = ${ $this_warp }[7]; + if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; } else { + $comment_hash_good{$this_comment} = $this_exp_id; + } + + #also save the entries which are ok, but not best quality + if (($this_quality == 0) && ($this_state ne 'drop') ) { $comment_hash{$this_comment} = $this_exp_id; } } - # Exclude any warps that are not stored in the comment_hash. + + # Exclude any warps that are not stored in the comment_hash_good. my @keep_warps = (); foreach my $this_warp (@{ $warps }) { my $this_comment = ${ $this_warp }[3]; my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash{$this_comment}))&& - ($comment_hash{$this_comment} == $this_exp_id)) { + if ((exists($comment_hash_good{$this_comment}))&& + ($comment_hash_good{$this_comment} == $this_exp_id)) { push @keep_warps, $this_warp; + my $this_date = ${ $this_warp }[2]; + my $chunk_name = ${ $this_warp }[6]; } else { - print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n"; + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; } } - # We are attempting to do the missing diffs, so reverse the list of retained warps. - # Good objects with all visits will be skipped due to the duplicate check. - # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. - @{ $warps } = reverse @keep_warps; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warps } + 1) % 2 != 0) { - print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; - if ($#{ $warps} + 1 == 1) { - print STDERR ": I can do no diffs with only one exposure.\n"; - next; + @{ $warps } = @keep_warps; + my $nwarps = ($#{ $warps } + 1); + + ################################ + #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs + #Conversely, less than 3 warps might result in dirty diffs + ################################ + if($nwarps >= 3) { + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warps } = reverse @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; + if ($#{ $warps} + 1 == 1) { + print STDERR ": I can do no diffs with only one exposure.\n"; + next; + } + else { + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } } - else { - my $rejected_warp = pop @{ $warps }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + + while ($#{ $warps } > -1) { + # The array is sorted in pairs of input/template. + my $template_warp = shift @{ $warps }; + my $input_warp = shift @{ $warps }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } + + } else { + #Consider the case of on-the-fly dirty desperate diffs + #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old + + # Exclude any warps that are not stored in the more general comment_hash. + my @keep_warps = (); + foreach my $this_warp (@{ $warpsBQ }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ((exists($comment_hash{$this_comment}))&& + ($comment_hash{$this_comment} == $this_exp_id)) { + push @keep_warps, $this_warp; + my $this_date = ${ $this_warp }[2]; + my $chunk_name = ${ $this_warp }[6]; + } + else { + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; + } } - } + @{ $warpsBQ } = @keep_warps; + + if ($nwarps >= 2) { + #one regular diff has already been made. So, need only one more. Strip a good entry from the array + my $nrem = 0; + my @keep_warps = (); + foreach my $this_warp (@{ $warpsBQ }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + my $this_fwhm = ${ $this_warp }[7]; + if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { + $nrem++; + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; + } else { + push @keep_warps, $this_warp; + } + } + @{ $warpsBQ } = @keep_warps; + } + + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warpsBQ } = reverse @{ $warpsBQ }; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warpsBQ } + 1) % 2 != 0) { + print STDERR "desp_diff_queue: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; + my $rejected_warp = pop @{ $warpsBQ }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } + my $nwarpsBQ = ($#{$warpsBQ} + 1); - while ($#{ $warps } > -1) { - # In this mode, the array is sorted in pairs of template/input, counter to the standard queue. - my $template_warp = shift @{ $warps }; - my $input_warp = shift @{ $warps }; - - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && + + while ($#{ $warpsBQ } > -1) { + # The array is sorted in pairs of input/template. + my $template_warp = shift @{ $warpsBQ }; + my $input_warp = shift @{ $warpsBQ }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; - } - next; - } - - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; - } - else { - print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } } - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; -# $cmd .= " -pretend "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "desp_diff_queue: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } + } } @@ -2041,8 +2399,4 @@ $metadata_out{nsDiffPotential} += $Npotential; $metadata_out{nsDiffQueued} += $Nqueued; -# if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { -# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; -# } - }