Index: trunk/ippScripts/scripts/nightly_science.pl =================================================================== --- trunk/ippScripts/scripts/nightly_science.pl (revision 42154) +++ trunk/ippScripts/scripts/nightly_science.pl (revision 42155) @@ -142,7 +142,4 @@ my %unique_filter_hash = (); -#set up the camera config -my $ipprcam = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up camera", $PS_EXIT_CONFIG_ERROR ); # IPP camera configuration - # Grab the configuration data. my $conf_cmd = "$ppConfigDump -camera $camera -dump-recipe NIGHTLY_SCIENCE -"; @@ -1394,29 +1391,4 @@ foreach my $target (sort (keys %science_config)) { if ($science_config{$target}{DIFFABLE} == 1) { - - my $config_reduction = "DEFAULT"; - if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};} - print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n"; - - my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use - &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; - - my $recipeData; - { - # Get the PSASTRO recipe - my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR); - } - $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR); - } - - my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); - print STDERR "execute_diffs: Max FWHM= $maxFWHM for $target\n"; - foreach my $filter (@filter_list) { $Npotential ++; @@ -1441,9 +1413,9 @@ print STDERR "execute_diffs: Target $target in filter $filter on $date has exposures and will be queued.\n"; } - diff_queue($date,$target,$filter,$maxFWHM,$pretend); + diff_queue($date,$target,$filter,$pretend); #Queue up desperate diffs if the conditions are right for them if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { - desperate_diff_singles($date,$target,$filter,$maxFWHM,$pretend); + desperate_diff_singles($date,$target,$filter,$pretend); } @@ -1487,32 +1459,8 @@ if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') { foreach my $target (sort (keys %science_config)) { - - my $config_reduction = "DEFAULT"; - if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};} - print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n"; - - my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use - &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; - - my $recipeData; - { - # Get the PSASTRO recipe - my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR); - } - $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR); - } - - my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); - # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now. if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { foreach my $filter (@filter_list) { - desperate_diff_queue($date,$target,$filter,$maxFWHM,$pretend); + desperate_diff_queue($date,$target,$filter,$pretend); } } @@ -1540,5 +1488,4 @@ my $target = shift; my $filter = shift; - my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -1560,5 +1507,4 @@ $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; - $input_sth .= " AND camProcessedExp.fwhm_major <= $maxFWHM "; $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs "; @@ -1693,5 +1639,4 @@ my $target = shift; my $filter = shift; - my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -1749,5 +1694,5 @@ my $this_chunk = shift @{ $object_row }; - my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)),camProcessedExp.fwhm_major FROM "; + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM "; $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; @@ -1755,9 +1700,7 @@ my $warps = $db->selectall_arrayref( $input_sth ); - my $warpsBQ = $db->selectall_arrayref( $input_sth );; # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. my %comment_hash = (); - my %comment_hash_good = (); foreach my $this_warp (@{ $warps }) { my $this_comment = ${ $this_warp }[3]; @@ -1765,25 +1708,18 @@ my $this_quality = ${ $this_warp }[5]; my $this_state = ${ $this_warp }[4]; - my $this_fwhm = ${ $this_warp }[7]; - if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { - print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; + if (($this_quality != 0) || ($this_state eq 'drop') ) { + print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; } else { - $comment_hash_good{$this_comment} = $this_exp_id; - } - - #also save the entries which are ok, but not best quality - if (($this_quality == 0) && ($this_state ne 'drop') ) { $comment_hash{$this_comment} = $this_exp_id; } } - - # Exclude any warps that are not stored in the comment_hash_good. + # Exclude any warps that are not stored in the comment_hash. my @keep_warps = (); foreach my $this_warp (@{ $warps }) { my $this_comment = ${ $this_warp }[3]; my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash_good{$this_comment}))&& - ($comment_hash_good{$this_comment} == $this_exp_id)) { + if ((exists($comment_hash{$this_comment}))&& + ($comment_hash{$this_comment} == $this_exp_id)) { push @keep_warps, $this_warp; $this_date = ${ $this_warp }[2]; @@ -1795,6 +1731,5 @@ } @{ $warps } = @keep_warps; - my $nwarps = ($#{ $warps } + 1); - + #find the time of the most recent exposure in that chunk my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' "; @@ -1805,4 +1740,5 @@ #compute and store some stats for potential on-the-fly desperate diffs + my $nwarps = ($#{ $warps } + 1); my $timediff = 0; my $now=DateTime->now; @@ -1816,243 +1752,99 @@ } } + + #Consider the special case of on-the-fly desperate diffs + #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old + if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { + print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; + next; + } + if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { + next; + } + + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warps } = reverse @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } + + while ($#{ $warps } > -1) { + # The array is sorted in pairs of input/template. + my $template_warp = shift @{ $warps }; + my $input_warp = shift @{ $warps }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } - ################################ - #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs - #Conversely, less than 3 good quality warps might result in dirty diffs - ################################ - if($nwarps >= 3) { - #Consider the special case of on-the-fly desperate diffs - #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old - if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { - print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; - next; - } - if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { - next; - } - - # We are attempting to do the missing diffs, so reverse the list of retained warps. - # Good objects with all visits will be skipped due to the duplicate check. - # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. - @{ $warps } = reverse @keep_warps; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warps } + 1) % 2 != 0) { - print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; - my $rejected_warp = pop @{ $warps }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; } - while ($#{ $warps } > -1) { - # The array is sorted in pairs of input/template. - my $template_warp = shift @{ $warps }; - my $input_warp = shift @{ $warps }; - - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && - ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; - } - next; - } - - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; - } - else { - print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - } - - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "desp_diff_singles: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } - } - - } else { - #Consider the case of on-the-fly dirty desperate diffs - #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old - - # Exclude any warps that are not stored in the more general comment_hash. - my @keep_warps = (); - foreach my $this_warp (@{ $warpsBQ }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash{$this_comment}))&& - ($comment_hash{$this_comment} == $this_exp_id)) { - push @keep_warps, $this_warp; - $this_date = ${ $this_warp }[2]; - $chunk_name = ${ $this_warp }[6]; - } - else { - print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; - } + if (defined($pretend)) { + $cmd .= ' -pretend '; } - @{ $warpsBQ } = @keep_warps; - - if ($nwarps >= 2) { - #one regular diff has already been made. So, need only one more. Strip a good entry from the array - my $nrem = 0; - my @keep_warps = (); - foreach my $this_warp (@{ $warpsBQ }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - my $this_fwhm = ${ $this_warp }[7]; - if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { - $nrem++; - print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; - } else { - push @keep_warps, $this_warp; - } - } - @{ $warpsBQ } = @keep_warps; - } - - # We are attempting to do the missing diffs, so reverse the list of retained warps. - # Good objects with all visits will be skipped due to the duplicate check. - # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. - @{ $warpsBQ } = reverse @{ $warpsBQ }; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warpsBQ } + 1) % 2 != 0) { - print STDERR "desp_diff_singles: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; - my $rejected_warp = pop @{ $warpsBQ }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; - } - my $nwarpsBQ = ($#{$warpsBQ} + 1); - - - if (($nwarpsBQ >= 2) && ($timediff < $desdiffdt)) { - print STDERR "desp_diff_singles: There are potential dirty desperate diffs to be done, but the time criterium is not met.\n"; - next; - } - if (($nwarpsBQ <= 1) || ($timediff < $desdiffdt)) { - next; - } - - while ($#{ $warpsBQ } > -1) { - # The array is sorted in pairs of input/template. - my $template_warp = shift @{ $warpsBQ }; - my $input_warp = shift @{ $warpsBQ }; - - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && - ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; - } - next; - } - - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; - } - else { - print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - } - - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "desp_diff_singles: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_singles: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; } - - + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } } } @@ -2066,5 +1858,4 @@ my $target = shift; my $filter = shift; - my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -2120,5 +1911,5 @@ my $this_chunk = shift @{ $object_row }; - my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,camProcessedExp.fwhm_major FROM "; + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM "; $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; @@ -2126,9 +1917,7 @@ my $warps = $db->selectall_arrayref( $input_sth ); - my $warpsBQ = $db->selectall_arrayref( $input_sth );; # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. my %comment_hash = (); - my %comment_hash_good = (); foreach my $this_warp (@{ $warps }) { my $this_comment = ${ $this_warp }[3]; @@ -2136,262 +1925,115 @@ my $this_quality = ${ $this_warp }[5]; my $this_state = ${ $this_warp }[4]; - my $this_fwhm = ${ $this_warp }[7]; - if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { - print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; + if (($this_quality != 0) || ($this_state eq 'drop') ) { + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; } else { - $comment_hash_good{$this_comment} = $this_exp_id; - } - - #also save the entries which are ok, but not best quality - if (($this_quality == 0) && ($this_state ne 'drop') ) { $comment_hash{$this_comment} = $this_exp_id; } } - - # Exclude any warps that are not stored in the comment_hash_good. + # Exclude any warps that are not stored in the comment_hash. my @keep_warps = (); foreach my $this_warp (@{ $warps }) { my $this_comment = ${ $this_warp }[3]; my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash_good{$this_comment}))&& - ($comment_hash_good{$this_comment} == $this_exp_id)) { + if ((exists($comment_hash{$this_comment}))&& + ($comment_hash{$this_comment} == $this_exp_id)) { push @keep_warps, $this_warp; - my $this_date = ${ $this_warp }[2]; - my $chunk_name = ${ $this_warp }[6]; } else { - print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n"; } } - @{ $warps } = @keep_warps; - my $nwarps = ($#{ $warps } + 1); - - ################################ - #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs - #Conversely, less than 3 warps might result in dirty diffs - ################################ - if($nwarps >= 3) { - # We are attempting to do the missing diffs, so reverse the list of retained warps. - # Good objects with all visits will be skipped due to the duplicate check. - # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. - @{ $warps } = reverse @keep_warps; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warps } + 1) % 2 != 0) { - print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; - if ($#{ $warps} + 1 == 1) { - print STDERR ": I can do no diffs with only one exposure.\n"; - next; - } - else { - my $rejected_warp = pop @{ $warps }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; - } + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warps } = reverse @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; + if ($#{ $warps} + 1 == 1) { + print STDERR ": I can do no diffs with only one exposure.\n"; + next; } + else { + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } + } - while ($#{ $warps } > -1) { - # The array is sorted in pairs of input/template. - my $template_warp = shift @{ $warps }; - my $input_warp = shift @{ $warps }; - - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && + while ($#{ $warps } > -1) { + # In this mode, the array is sorted in pairs of template/input, counter to the standard queue. + my $template_warp = shift @{ $warps }; + my $input_warp = shift @{ $warps }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; - } - next; - } - - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; - } - else { - print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - } - - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "desp_diff_queue: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } - } - - } else { - #Consider the case of on-the-fly dirty desperate diffs - #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old - - # Exclude any warps that are not stored in the more general comment_hash. - my @keep_warps = (); - foreach my $this_warp (@{ $warpsBQ }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash{$this_comment}))&& - ($comment_hash{$this_comment} == $this_exp_id)) { - push @keep_warps, $this_warp; - my $this_date = ${ $this_warp }[2]; - my $chunk_name = ${ $this_warp }[6]; - } - else { - print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; - } + print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; } - @{ $warpsBQ } = @keep_warps; - - if ($nwarps >= 2) { - #one regular diff has already been made. So, need only one more. Strip a good entry from the array - my $nrem = 0; - my @keep_warps = (); - foreach my $this_warp (@{ $warpsBQ }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - my $this_fwhm = ${ $this_warp }[7]; - if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { - $nrem++; - print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; - } else { - push @keep_warps, $this_warp; - } - } - @{ $warpsBQ } = @keep_warps; + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; } - - # We are attempting to do the missing diffs, so reverse the list of retained warps. - # Good objects with all visits will be skipped due to the duplicate check. - # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. - @{ $warpsBQ } = reverse @{ $warpsBQ }; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warpsBQ } + 1) % 2 != 0) { - print STDERR "desp_diff_queue: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; - my $rejected_warp = pop @{ $warpsBQ }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; - } - my $nwarpsBQ = ($#{$warpsBQ} + 1); - - - while ($#{ $warpsBQ } > -1) { - # The array is sorted in pairs of input/template. - my $template_warp = shift @{ $warpsBQ }; - my $input_warp = shift @{ $warpsBQ }; - - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && - ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; - } - next; - } - - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; - } - else { - print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - } - - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "desp_diff_queue: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; } + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# $cmd .= " -pretend "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } - + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } } } @@ -2399,4 +2041,8 @@ $metadata_out{nsDiffPotential} += $Npotential; $metadata_out{nsDiffQueued} += $Nqueued; +# if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { +# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; +# } + }