Index: trunk/ippScripts/scripts/nightly_science.pl =================================================================== --- trunk/ippScripts/scripts/nightly_science.pl (revision 42229) +++ trunk/ippScripts/scripts/nightly_science.pl (revision 42253) @@ -1443,5 +1443,5 @@ diff_queue($date,$target,$filter,$maxFWHM,$pretend); - #Queue up desperate diffs if the conditions are right for them + #Queue up desperate diffs if the conditions are right for them if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { desperate_diff_singles($date,$target,$filter,$maxFWHM,$pretend); @@ -1919,138 +1919,138 @@ } else { - print STDERR "desp_diff_singles: There is a possibility for dirty diffs to be made\n"; + print STDERR "desp_diff_singles: There is a possibility for dirty diffs to be made\n"; #Consider the case of on-the-fly dirty desperate diffs #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old -# # Exclude any warps that are not stored in the more general comment_hash. -# my @keep_warps = (); -# foreach my $this_warp (@{ $warpsBQ }) { -# my $this_comment = ${ $this_warp }[3]; -# my $this_exp_id = ${ $this_warp }[0]; -# if ((exists($comment_hash{$this_comment}))&& -# ($comment_hash{$this_comment} == $this_exp_id)) { -# push @keep_warps, $this_warp; -# $this_date = ${ $this_warp }[2]; -# $chunk_name = ${ $this_warp }[6]; -# } -# else { -# print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; -# } -# } -# @{ $warpsBQ } = @keep_warps; -# -# if ($nwarps >= 2) { -# #one regular diff has already been made. So, need only one more. Strip a good entry from the array -# my $nrem = 0; -# my @keep_warps = (); -# foreach my $this_warp (@{ $warpsBQ }) { -# my $this_comment = ${ $this_warp }[3]; -# my $this_exp_id = ${ $this_warp }[0]; -# my $this_fwhm = ${ $this_warp }[7]; -# if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { -# $nrem++; -# print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; -# } else { -# push @keep_warps, $this_warp; -# } -# } -# @{ $warpsBQ } = @keep_warps; -# } -# -# # We are attempting to do the missing diffs, so reverse the list of retained warps. -# # Good objects with all visits will be skipped due to the duplicate check. -# # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. -# @{ $warpsBQ } = reverse @{ $warpsBQ }; -# -# # Exclude the last entry if we do not have an even number of warps. -# if (($#{ $warpsBQ } + 1) % 2 != 0) { -# print STDERR "desp_diff_singles: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; -# my $rejected_warp = pop @{ $warpsBQ }; -# my $rejected_exp_id = ${ $rejected_warp }[0]; -# print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; -# } -# my $nwarpsBQ = ($#{$warpsBQ} + 1); -# -# -# if (($nwarpsBQ >= 2) && ($timediff < $desdiffdt)) { -# print STDERR "desp_diff_singles: There are potential dirty desperate diffs to be done, but the time criterium is not met.\n"; -# next; -# } -# if (($nwarpsBQ <= 1) || ($timediff < $desdiffdt)) { -# next; -# } -# -# while ($#{ $warpsBQ } > -1) { -# # The array is sorted in pairs of input/template. -# my $template_warp = shift @{ $warpsBQ }; -# my $input_warp = shift @{ $warpsBQ }; +# # Exclude any warps that are not stored in the more general comment_hash. +# my @keep_warps = (); +# foreach my $this_warp (@{ $warpsBQ }) { +# my $this_comment = ${ $this_warp }[3]; +# my $this_exp_id = ${ $this_warp }[0]; +# if ((exists($comment_hash{$this_comment}))&& +# ($comment_hash{$this_comment} == $this_exp_id)) { +# push @keep_warps, $this_warp; +# $this_date = ${ $this_warp }[2]; +# $chunk_name = ${ $this_warp }[6]; +# } +# else { +# print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; +# } +# } +# @{ $warpsBQ } = @keep_warps; +# +# if ($nwarps >= 2) { +# #one regular diff has already been made. So, need only one more. Strip a good entry from the array +# my $nrem = 0; +# my @keep_warps = (); +# foreach my $this_warp (@{ $warpsBQ }) { +# my $this_comment = ${ $this_warp }[3]; +# my $this_exp_id = ${ $this_warp }[0]; +# my $this_fwhm = ${ $this_warp }[7]; +# if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { +# $nrem++; +# print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; +# } else { +# push @keep_warps, $this_warp; +# } +# } +# @{ $warpsBQ } = @keep_warps; +# } +# +# # We are attempting to do the missing diffs, so reverse the list of retained warps. +# # Good objects with all visits will be skipped due to the duplicate check. +# # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. +# @{ $warpsBQ } = reverse @{ $warpsBQ }; +# +# # Exclude the last entry if we do not have an even number of warps. +# if (($#{ $warpsBQ } + 1) % 2 != 0) { +# print STDERR "desp_diff_singles: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; +# my $rejected_warp = pop @{ $warpsBQ }; +# my $rejected_exp_id = ${ $rejected_warp }[0]; +# print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; +# } +# my $nwarpsBQ = ($#{$warpsBQ} + 1); +# +# +# if (($nwarpsBQ >= 2) && ($timediff < $desdiffdt)) { +# print STDERR "desp_diff_singles: There are potential dirty desperate diffs to be done, but the time criterium is not met.\n"; +# next; +# } +# if (($nwarpsBQ <= 1) || ($timediff < $desdiffdt)) { +# next; +# } +# +# while ($#{ $warpsBQ } > -1) { +# # The array is sorted in pairs of input/template. +# my $template_warp = shift @{ $warpsBQ }; +# my $input_warp = shift @{ $warpsBQ }; # -# my $input_exp_id = ${ $input_warp }[0]; -# my $input_comment = ${ $input_warp }[3]; -# -# my $template_exp_id = ${ $template_warp }[0]; -# my $template_comment = ${ $template_warp }[3]; -# -# my $input_warp_id = ${ $input_warp }[1]; -# my $template_warp_id = ${ $template_warp }[1]; -# -# my $input_warp_state = ${ $input_warp }[4]; -# my $template_warp_state = ${ $template_warp }[4]; -# -# my $input_warp_camQuality = ${ $input_warp }[5]; -# my $template_warp_camQuality = ${ $template_warp }[5]; -# -# $Npotential++; -# -# unless (defined($input_warp_id) && defined($template_warp_id) && -# ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { -# print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; -# if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { -# # This should now never be reached. -# # CZW: Trigger backup plan here? Or simply set up framework? -# print STDERR " ...but this is due to a camera stage astrometry quality\n"; -# $Npotential--; -# } -# next; -# } -# -# if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { -# $Nqueued++; -# print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; -# next; -# } -# else { -# print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; -# } -# -# my $cmd = "$difftool -dbname $dbname -definewarpwarp "; -# $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; -# $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. -# $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures -# $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; -# $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; -# if (defined($reduction)) { -# $cmd .= " -set_reduction $reduction "; -# } -# -# if (defined($pretend)) { -# $cmd .= ' -pretend '; -# } -# if ($debug == 1) { -# $cmd .= ' -pretend '; -# print STDERR "desp_diff_singles: $cmd\n"; -# print STDERR " $input_warp_id $template_warp_id\n"; -# } -# -# if (($debug == 0)&&(!defined($pretend))) { -# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run ( command => $cmd, verbose => $verbose ); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); -# } -# $Nqueued++; -# } -# } +# my $input_exp_id = ${ $input_warp }[0]; +# my $input_comment = ${ $input_warp }[3]; +# +# my $template_exp_id = ${ $template_warp }[0]; +# my $template_comment = ${ $template_warp }[3]; +# +# my $input_warp_id = ${ $input_warp }[1]; +# my $template_warp_id = ${ $template_warp }[1]; +# +# my $input_warp_state = ${ $input_warp }[4]; +# my $template_warp_state = ${ $template_warp }[4]; +# +# my $input_warp_camQuality = ${ $input_warp }[5]; +# my $template_warp_camQuality = ${ $template_warp }[5]; +# +# $Npotential++; +# +# unless (defined($input_warp_id) && defined($template_warp_id) && +# ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { +# print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; +# if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { +# # This should now never be reached. +# # CZW: Trigger backup plan here? Or simply set up framework? +# print STDERR " ...but this is due to a camera stage astrometry quality\n"; +# $Npotential--; +# } +# next; +# } +# +# if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { +# $Nqueued++; +# print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; +# next; +# } +# else { +# print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; +# } +# +# my $cmd = "$difftool -dbname $dbname -definewarpwarp "; +# $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; +# $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. +# $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures +# $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; +# $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# if (defined($reduction)) { +# $cmd .= " -set_reduction $reduction "; +# } +# +# if (defined($pretend)) { +# $cmd .= ' -pretend '; +# } +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# print STDERR "desp_diff_singles: $cmd\n"; +# print STDERR " $input_warp_id $template_warp_id\n"; +# } +# +# if (($debug == 0)&&(!defined($pretend))) { +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $cmd, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# $Nqueued++; +# } +# } @@ -2285,116 +2285,116 @@ # print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; # } -# } -# @{ $warpsBQ } = @keep_warps; -# -# if ($nwarps >= 2) { -# #one regular diff has already been made. So, need only one more. Strip a good entry from the array -# my $nrem = 0; -# my @keep_warps = (); -# foreach my $this_warp (@{ $warpsBQ }) { -# my $this_comment = ${ $this_warp }[3]; -# my $this_exp_id = ${ $this_warp }[0]; -# my $this_fwhm = ${ $this_warp }[7]; -# if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { -# $nrem++; -# print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; -# } else { -# push @keep_warps, $this_warp; -# } -# } -# @{ $warpsBQ } = @keep_warps; -# } -# -# # We are attempting to do the missing diffs, so reverse the list of retained warps. -# # Good objects with all visits will be skipped due to the duplicate check. -# # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. -# @{ $warpsBQ } = reverse @{ $warpsBQ }; -# -# # Exclude the last entry if we do not have an even number of warps. -# if (($#{ $warpsBQ } + 1) % 2 != 0) { -# print STDERR "desp_diff_queue: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; -# my $rejected_warp = pop @{ $warpsBQ }; -# my $rejected_exp_id = ${ $rejected_warp }[0]; -# print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; -# } -# my $nwarpsBQ = ($#{$warpsBQ} + 1); -# -# -# while ($#{ $warpsBQ } > -1) { -# # The array is sorted in pairs of input/template. -# my $template_warp = shift @{ $warpsBQ }; -# my $input_warp = shift @{ $warpsBQ }; +# } +# @{ $warpsBQ } = @keep_warps; +# +# if ($nwarps >= 2) { +# #one regular diff has already been made. So, need only one more. Strip a good entry from the array +# my $nrem = 0; +# my @keep_warps = (); +# foreach my $this_warp (@{ $warpsBQ }) { +# my $this_comment = ${ $this_warp }[3]; +# my $this_exp_id = ${ $this_warp }[0]; +# my $this_fwhm = ${ $this_warp }[7]; +# if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { +# $nrem++; +# print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; +# } else { +# push @keep_warps, $this_warp; +# } +# } +# @{ $warpsBQ } = @keep_warps; +# } +# +# # We are attempting to do the missing diffs, so reverse the list of retained warps. +# # Good objects with all visits will be skipped due to the duplicate check. +# # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. +# @{ $warpsBQ } = reverse @{ $warpsBQ }; +# +# # Exclude the last entry if we do not have an even number of warps. +# if (($#{ $warpsBQ } + 1) % 2 != 0) { +# print STDERR "desp_diff_queue: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; +# my $rejected_warp = pop @{ $warpsBQ }; +# my $rejected_exp_id = ${ $rejected_warp }[0]; +# print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; +# } +# my $nwarpsBQ = ($#{$warpsBQ} + 1); +# +# +# while ($#{ $warpsBQ } > -1) { +# # The array is sorted in pairs of input/template. +# my $template_warp = shift @{ $warpsBQ }; +# my $input_warp = shift @{ $warpsBQ }; # -# my $input_exp_id = ${ $input_warp }[0]; -# my $input_comment = ${ $input_warp }[3]; -# -# my $template_exp_id = ${ $template_warp }[0]; -# my $template_comment = ${ $template_warp }[3]; -# -# my $input_warp_id = ${ $input_warp }[1]; -# my $template_warp_id = ${ $template_warp }[1]; -# -# my $input_warp_state = ${ $input_warp }[4]; -# my $template_warp_state = ${ $template_warp }[4]; -# -# my $input_warp_camQuality = ${ $input_warp }[5]; -# my $template_warp_camQuality = ${ $template_warp }[5]; -# -# $Npotential++; -# -# unless (defined($input_warp_id) && defined($template_warp_id) && -# ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { -# print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; -# if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { -# # This should now never be reached. -# # CZW: Trigger backup plan here? Or simply set up framework? -# print STDERR " ...but this is due to a camera stage astrometry quality\n"; -# $Npotential--; -# } -# next; -# } -# -# if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { -# $Nqueued++; -# print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; -# next; -# } -# else { -# print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; -# } -# -# my $cmd = "$difftool -dbname $dbname -definewarpwarp "; -# $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; -# $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. -# $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures -# $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; -# $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; -# if (defined($reduction)) { -# $cmd .= " -set_reduction $reduction "; -# } -# -# if (defined($pretend)) { -# $cmd .= ' -pretend '; -# } -# if ($debug == 1) { -# $cmd .= ' -pretend '; -# print STDERR "desp_diff_queue: $cmd\n"; -# print STDERR " $input_warp_id $template_warp_id\n"; -# } -# -# if (($debug == 0)&&(!defined($pretend))) { -# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run ( command => $cmd, verbose => $verbose ); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); -# } -# $Nqueued++; -# } -# } - - - - } +# my $input_exp_id = ${ $input_warp }[0]; +# my $input_comment = ${ $input_warp }[3]; +# +# my $template_exp_id = ${ $template_warp }[0]; +# my $template_comment = ${ $template_warp }[3]; +# +# my $input_warp_id = ${ $input_warp }[1]; +# my $template_warp_id = ${ $template_warp }[1]; +# +# my $input_warp_state = ${ $input_warp }[4]; +# my $template_warp_state = ${ $template_warp }[4]; +# +# my $input_warp_camQuality = ${ $input_warp }[5]; +# my $template_warp_camQuality = ${ $template_warp }[5]; +# +# $Npotential++; +# +# unless (defined($input_warp_id) && defined($template_warp_id) && +# ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { +# print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; +# if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { +# # This should now never be reached. +# # CZW: Trigger backup plan here? Or simply set up framework? +# print STDERR " ...but this is due to a camera stage astrometry quality\n"; +# $Npotential--; +# } +# next; +# } +# +# if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { +# $Nqueued++; +# print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; +# next; +# } +# else { +# print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; +# } +# +# my $cmd = "$difftool -dbname $dbname -definewarpwarp "; +# $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; +# $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. +# $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures +# $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; +# $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# if (defined($reduction)) { +# $cmd .= " -set_reduction $reduction "; +# } +# +# if (defined($pretend)) { +# $cmd .= ' -pretend '; +# } +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# print STDERR "desp_diff_queue: $cmd\n"; +# print STDERR " $input_warp_id $template_warp_id\n"; +# } +# +# if (($debug == 0)&&(!defined($pretend))) { +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $cmd, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# $Nqueued++; +# } +# } + + + + } } }