Changeset 42155
- Timestamp:
- Apr 4, 2022, 2:59:53 PM (4 years ago)
- Location:
- trunk/ippScripts/scripts
- Files:
-
- 2 edited
-
camera_exp.pl (modified) (1 diff)
-
nightly_science.pl (modified) (18 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/camera_exp.pl
r42154 r42155 178 178 179 179 # Determine if FWHM is too large to bother continuing. 180 my $maxFWHM = 30.;180 my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); 181 181 if ($maxFWHM) { 182 182 my $expFWHM; -
trunk/ippScripts/scripts/nightly_science.pl
r42154 r42155 142 142 my %unique_filter_hash = (); 143 143 144 #set up the camera config145 my $ipprcam = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up camera", $PS_EXIT_CONFIG_ERROR ); # IPP camera configuration146 147 144 # Grab the configuration data. 148 145 my $conf_cmd = "$ppConfigDump -camera $camera -dump-recipe NIGHTLY_SCIENCE -"; … … 1394 1391 foreach my $target (sort (keys %science_config)) { 1395 1392 if ($science_config{$target}{DIFFABLE} == 1) { 1396 1397 my $config_reduction = "DEFAULT";1398 if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};}1399 print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n";1400 1401 my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use1402 &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;1403 1404 my $recipeData;1405 {1406 # Get the PSASTRO recipe1407 my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";1408 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =1409 run(command => $command);1410 unless ($success) {1411 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);1412 &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR);1413 }1414 $recipeData = $mdcParser->parse(join "", @$stdout_buf) or1415 &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR);1416 }1417 1418 my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM');1419 print STDERR "execute_diffs: Max FWHM= $maxFWHM for $target\n";1420 1421 1393 foreach my $filter (@filter_list) { 1422 1394 $Npotential ++; … … 1441 1413 print STDERR "execute_diffs: Target $target in filter $filter on $date has exposures and will be queued.\n"; 1442 1414 } 1443 diff_queue($date,$target,$filter,$ maxFWHM,$pretend);1415 diff_queue($date,$target,$filter,$pretend); 1444 1416 1445 1417 #Queue up desperate diffs if the conditions are right for them 1446 1418 if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { 1447 desperate_diff_singles($date,$target,$filter,$ maxFWHM,$pretend);1419 desperate_diff_singles($date,$target,$filter,$pretend); 1448 1420 } 1449 1421 … … 1487 1459 if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') { 1488 1460 foreach my $target (sort (keys %science_config)) { 1489 1490 my $config_reduction = "DEFAULT";1491 if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};}1492 print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n";1493 1494 my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use1495 &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;1496 1497 my $recipeData;1498 {1499 # Get the PSASTRO recipe1500 my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";1501 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =1502 run(command => $command);1503 unless ($success) {1504 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);1505 &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR);1506 }1507 $recipeData = $mdcParser->parse(join "", @$stdout_buf) or1508 &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR);1509 }1510 1511 my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM');1512 1513 1461 # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now. 1514 1462 if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { 1515 1463 foreach my $filter (@filter_list) { 1516 desperate_diff_queue($date,$target,$filter,$ maxFWHM,$pretend);1464 desperate_diff_queue($date,$target,$filter,$pretend); 1517 1465 } 1518 1466 } … … 1540 1488 my $target = shift; 1541 1489 my $filter = shift; 1542 my $maxFWHM = shift;1543 1490 my $pretend = shift; 1544 1491 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); … … 1560 1507 $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; 1561 1508 $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; 1562 $input_sth .= " AND camProcessedExp.fwhm_major <= $maxFWHM ";1563 1509 $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs "; 1564 1510 … … 1693 1639 my $target = shift; 1694 1640 my $filter = shift; 1695 my $maxFWHM = shift;1696 1641 my $pretend = shift; 1697 1642 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); … … 1749 1694 my $this_chunk = shift @{ $object_row }; 1750 1695 1751 my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) ,camProcessedExp.fwhm_majorFROM ";1696 my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM "; 1752 1697 $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; 1753 1698 $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; … … 1755 1700 1756 1701 my $warps = $db->selectall_arrayref( $input_sth ); 1757 my $warpsBQ = $db->selectall_arrayref( $input_sth );;1758 1702 1759 1703 # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. 1760 1704 my %comment_hash = (); 1761 my %comment_hash_good = ();1762 1705 foreach my $this_warp (@{ $warps }) { 1763 1706 my $this_comment = ${ $this_warp }[3]; … … 1765 1708 my $this_quality = ${ $this_warp }[5]; 1766 1709 my $this_state = ${ $this_warp }[4]; 1767 my $this_fwhm = ${ $this_warp }[7]; 1768 if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { 1769 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; 1710 if (($this_quality != 0) || ($this_state eq 'drop') ) { 1711 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; 1770 1712 } 1771 1713 else { 1772 $comment_hash_good{$this_comment} = $this_exp_id;1773 }1774 1775 #also save the entries which are ok, but not best quality1776 if (($this_quality == 0) && ($this_state ne 'drop') ) {1777 1714 $comment_hash{$this_comment} = $this_exp_id; 1778 1715 } 1779 1716 } 1780 1781 # Exclude any warps that are not stored in the comment_hash_good. 1717 # Exclude any warps that are not stored in the comment_hash. 1782 1718 my @keep_warps = (); 1783 1719 foreach my $this_warp (@{ $warps }) { 1784 1720 my $this_comment = ${ $this_warp }[3]; 1785 1721 my $this_exp_id = ${ $this_warp }[0]; 1786 if ((exists($comment_hash _good{$this_comment}))&&1787 ($comment_hash _good{$this_comment} == $this_exp_id)) {1722 if ((exists($comment_hash{$this_comment}))&& 1723 ($comment_hash{$this_comment} == $this_exp_id)) { 1788 1724 push @keep_warps, $this_warp; 1789 1725 $this_date = ${ $this_warp }[2]; … … 1795 1731 } 1796 1732 @{ $warps } = @keep_warps; 1797 my $nwarps = ($#{ $warps } + 1); 1798 1733 1799 1734 #find the time of the most recent exposure in that chunk 1800 1735 my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' "; … … 1805 1740 1806 1741 #compute and store some stats for potential on-the-fly desperate diffs 1742 my $nwarps = ($#{ $warps } + 1); 1807 1743 my $timediff = 0; 1808 1744 my $now=DateTime->now; … … 1816 1752 } 1817 1753 } 1754 1755 #Consider the special case of on-the-fly desperate diffs 1756 #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old 1757 if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { 1758 print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; 1759 next; 1760 } 1761 if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { 1762 next; 1763 } 1764 1765 # We are attempting to do the missing diffs, so reverse the list of retained warps. 1766 # Good objects with all visits will be skipped due to the duplicate check. 1767 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 1768 @{ $warps } = reverse @keep_warps; 1769 1770 # Exclude the last entry if we do not have an even number of warps. 1771 if (($#{ $warps } + 1) % 2 != 0) { 1772 print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; 1773 my $rejected_warp = pop @{ $warps }; 1774 my $rejected_exp_id = ${ $rejected_warp }[0]; 1775 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 1776 } 1777 1778 while ($#{ $warps } > -1) { 1779 # The array is sorted in pairs of input/template. 1780 my $template_warp = shift @{ $warps }; 1781 my $input_warp = shift @{ $warps }; 1782 1783 my $input_exp_id = ${ $input_warp }[0]; 1784 my $input_comment = ${ $input_warp }[3]; 1785 1786 my $template_exp_id = ${ $template_warp }[0]; 1787 my $template_comment = ${ $template_warp }[3]; 1788 1789 my $input_warp_id = ${ $input_warp }[1]; 1790 my $template_warp_id = ${ $template_warp }[1]; 1791 1792 my $input_warp_state = ${ $input_warp }[4]; 1793 my $template_warp_state = ${ $template_warp }[4]; 1794 1795 my $input_warp_camQuality = ${ $input_warp }[5]; 1796 my $template_warp_camQuality = ${ $template_warp }[5]; 1797 1798 $Npotential++; 1799 1800 unless (defined($input_warp_id) && defined($template_warp_id) && 1801 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 1802 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 1803 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 1804 # This should now never be reached. 1805 # CZW: Trigger backup plan here? Or simply set up framework? 1806 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 1807 $Npotential--; 1808 } 1809 next; 1810 } 1811 1812 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 1813 $Nqueued++; 1814 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 1815 next; 1816 } 1817 else { 1818 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 1819 } 1818 1820 1819 ################################ 1820 #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs 1821 #Conversely, less than 3 good quality warps might result in dirty diffs 1822 ################################ 1823 if($nwarps >= 3) { 1824 #Consider the special case of on-the-fly desperate diffs 1825 #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old 1826 if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { 1827 print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; 1828 next; 1829 } 1830 if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { 1831 next; 1832 } 1833 1834 # We are attempting to do the missing diffs, so reverse the list of retained warps. 1835 # Good objects with all visits will be skipped due to the duplicate check. 1836 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 1837 @{ $warps } = reverse @keep_warps; 1838 1839 # Exclude the last entry if we do not have an even number of warps. 1840 if (($#{ $warps } + 1) % 2 != 0) { 1841 print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; 1842 my $rejected_warp = pop @{ $warps }; 1843 my $rejected_exp_id = ${ $rejected_warp }[0]; 1844 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 1821 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 1822 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 1823 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 1824 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 1825 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 1826 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 1827 if (defined($reduction)) { 1828 $cmd .= " -set_reduction $reduction "; 1845 1829 } 1846 1830 1847 while ($#{ $warps } > -1) { 1848 # The array is sorted in pairs of input/template. 1849 my $template_warp = shift @{ $warps }; 1850 my $input_warp = shift @{ $warps }; 1851 1852 my $input_exp_id = ${ $input_warp }[0]; 1853 my $input_comment = ${ $input_warp }[3]; 1854 1855 my $template_exp_id = ${ $template_warp }[0]; 1856 my $template_comment = ${ $template_warp }[3]; 1857 1858 my $input_warp_id = ${ $input_warp }[1]; 1859 my $template_warp_id = ${ $template_warp }[1]; 1860 1861 my $input_warp_state = ${ $input_warp }[4]; 1862 my $template_warp_state = ${ $template_warp }[4]; 1863 1864 my $input_warp_camQuality = ${ $input_warp }[5]; 1865 my $template_warp_camQuality = ${ $template_warp }[5]; 1866 1867 $Npotential++; 1868 1869 unless (defined($input_warp_id) && defined($template_warp_id) && 1870 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 1871 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 1872 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 1873 # This should now never be reached. 1874 # CZW: Trigger backup plan here? Or simply set up framework? 1875 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 1876 $Npotential--; 1877 } 1878 next; 1879 } 1880 1881 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 1882 $Nqueued++; 1883 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 1884 next; 1885 } 1886 else { 1887 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 1888 } 1889 1890 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 1891 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 1892 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 1893 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 1894 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 1895 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 1896 if (defined($reduction)) { 1897 $cmd .= " -set_reduction $reduction "; 1898 } 1899 1900 if (defined($pretend)) { 1901 $cmd .= ' -pretend '; 1902 } 1903 if ($debug == 1) { 1904 $cmd .= ' -pretend '; 1905 print STDERR "desp_diff_singles: $cmd\n"; 1906 print STDERR " $input_warp_id $template_warp_id\n"; 1907 } 1908 1909 if (($debug == 0)&&(!defined($pretend))) { 1910 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1911 run ( command => $cmd, verbose => $verbose ); 1912 unless ($success) { 1913 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1914 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 1915 } 1916 $Nqueued++; 1917 } 1918 } 1919 1920 } else { 1921 #Consider the case of on-the-fly dirty desperate diffs 1922 #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old 1923 1924 # Exclude any warps that are not stored in the more general comment_hash. 1925 my @keep_warps = (); 1926 foreach my $this_warp (@{ $warpsBQ }) { 1927 my $this_comment = ${ $this_warp }[3]; 1928 my $this_exp_id = ${ $this_warp }[0]; 1929 if ((exists($comment_hash{$this_comment}))&& 1930 ($comment_hash{$this_comment} == $this_exp_id)) { 1931 push @keep_warps, $this_warp; 1932 $this_date = ${ $this_warp }[2]; 1933 $chunk_name = ${ $this_warp }[6]; 1934 } 1935 else { 1936 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; 1937 } 1831 if (defined($pretend)) { 1832 $cmd .= ' -pretend '; 1938 1833 } 1939 @{ $warpsBQ } = @keep_warps; 1940 1941 if ($nwarps >= 2) { 1942 #one regular diff has already been made. So, need only one more. Strip a good entry from the array 1943 my $nrem = 0; 1944 my @keep_warps = (); 1945 foreach my $this_warp (@{ $warpsBQ }) { 1946 my $this_comment = ${ $this_warp }[3]; 1947 my $this_exp_id = ${ $this_warp }[0]; 1948 my $this_fwhm = ${ $this_warp }[7]; 1949 if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { 1950 $nrem++; 1951 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; 1952 } else { 1953 push @keep_warps, $this_warp; 1954 } 1955 } 1956 @{ $warpsBQ } = @keep_warps; 1957 } 1958 1959 # We are attempting to do the missing diffs, so reverse the list of retained warps. 1960 # Good objects with all visits will be skipped due to the duplicate check. 1961 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 1962 @{ $warpsBQ } = reverse @{ $warpsBQ }; 1963 1964 # Exclude the last entry if we do not have an even number of warps. 1965 if (($#{ $warpsBQ } + 1) % 2 != 0) { 1966 print STDERR "desp_diff_singles: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; 1967 my $rejected_warp = pop @{ $warpsBQ }; 1968 my $rejected_exp_id = ${ $rejected_warp }[0]; 1969 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 1970 } 1971 my $nwarpsBQ = ($#{$warpsBQ} + 1); 1972 1973 1974 if (($nwarpsBQ >= 2) && ($timediff < $desdiffdt)) { 1975 print STDERR "desp_diff_singles: There are potential dirty desperate diffs to be done, but the time criterium is not met.\n"; 1976 next; 1977 } 1978 if (($nwarpsBQ <= 1) || ($timediff < $desdiffdt)) { 1979 next; 1980 } 1981 1982 while ($#{ $warpsBQ } > -1) { 1983 # The array is sorted in pairs of input/template. 1984 my $template_warp = shift @{ $warpsBQ }; 1985 my $input_warp = shift @{ $warpsBQ }; 1986 1987 my $input_exp_id = ${ $input_warp }[0]; 1988 my $input_comment = ${ $input_warp }[3]; 1989 1990 my $template_exp_id = ${ $template_warp }[0]; 1991 my $template_comment = ${ $template_warp }[3]; 1992 1993 my $input_warp_id = ${ $input_warp }[1]; 1994 my $template_warp_id = ${ $template_warp }[1]; 1995 1996 my $input_warp_state = ${ $input_warp }[4]; 1997 my $template_warp_state = ${ $template_warp }[4]; 1998 1999 my $input_warp_camQuality = ${ $input_warp }[5]; 2000 my $template_warp_camQuality = ${ $template_warp }[5]; 2001 2002 $Npotential++; 2003 2004 unless (defined($input_warp_id) && defined($template_warp_id) && 2005 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 2006 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 2007 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 2008 # This should now never be reached. 2009 # CZW: Trigger backup plan here? Or simply set up framework? 2010 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 2011 $Npotential--; 2012 } 2013 next; 2014 } 2015 2016 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 2017 $Nqueued++; 2018 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2019 next; 2020 } 2021 else { 2022 print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2023 } 2024 2025 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 2026 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 2027 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 2028 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 2029 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 2030 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 2031 if (defined($reduction)) { 2032 $cmd .= " -set_reduction $reduction "; 2033 } 2034 2035 if (defined($pretend)) { 2036 $cmd .= ' -pretend '; 2037 } 2038 if ($debug == 1) { 2039 $cmd .= ' -pretend '; 2040 print STDERR "desp_diff_singles: $cmd\n"; 2041 print STDERR " $input_warp_id $template_warp_id\n"; 2042 } 2043 2044 if (($debug == 0)&&(!defined($pretend))) { 2045 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2046 run ( command => $cmd, verbose => $verbose ); 2047 unless ($success) { 2048 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2049 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 2050 } 2051 $Nqueued++; 2052 } 1834 if ($debug == 1) { 1835 $cmd .= ' -pretend '; 1836 print STDERR "desp_diff_singles: $cmd\n"; 1837 print STDERR " $input_warp_id $template_warp_id\n"; 2053 1838 } 2054 1839 2055 2056 1840 if (($debug == 0)&&(!defined($pretend))) { 1841 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1842 run ( command => $cmd, verbose => $verbose ); 1843 unless ($success) { 1844 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1845 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 1846 } 1847 $Nqueued++; 1848 } 2057 1849 } 2058 1850 } … … 2066 1858 my $target = shift; 2067 1859 my $filter = shift; 2068 my $maxFWHM = shift;2069 1860 my $pretend = shift; 2070 1861 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); … … 2120 1911 my $this_chunk = shift @{ $object_row }; 2121 1912 2122 my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality ,camProcessedExp.fwhm_majorFROM ";1913 my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM "; 2123 1914 $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; 2124 1915 $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; … … 2126 1917 2127 1918 my $warps = $db->selectall_arrayref( $input_sth ); 2128 my $warpsBQ = $db->selectall_arrayref( $input_sth );;2129 1919 2130 1920 # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. 2131 1921 my %comment_hash = (); 2132 my %comment_hash_good = ();2133 1922 foreach my $this_warp (@{ $warps }) { 2134 1923 my $this_comment = ${ $this_warp }[3]; … … 2136 1925 my $this_quality = ${ $this_warp }[5]; 2137 1926 my $this_state = ${ $this_warp }[4]; 2138 my $this_fwhm = ${ $this_warp }[7]; 2139 if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { 2140 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; 1927 if (($this_quality != 0) || ($this_state eq 'drop') ) { 1928 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; 2141 1929 } 2142 1930 else { 2143 $comment_hash_good{$this_comment} = $this_exp_id;2144 }2145 2146 #also save the entries which are ok, but not best quality2147 if (($this_quality == 0) && ($this_state ne 'drop') ) {2148 1931 $comment_hash{$this_comment} = $this_exp_id; 2149 1932 } 2150 1933 } 2151 2152 # Exclude any warps that are not stored in the comment_hash_good. 1934 # Exclude any warps that are not stored in the comment_hash. 2153 1935 my @keep_warps = (); 2154 1936 foreach my $this_warp (@{ $warps }) { 2155 1937 my $this_comment = ${ $this_warp }[3]; 2156 1938 my $this_exp_id = ${ $this_warp }[0]; 2157 if ((exists($comment_hash _good{$this_comment}))&&2158 ($comment_hash _good{$this_comment} == $this_exp_id)) {1939 if ((exists($comment_hash{$this_comment}))&& 1940 ($comment_hash{$this_comment} == $this_exp_id)) { 2159 1941 push @keep_warps, $this_warp; 2160 my $this_date = ${ $this_warp }[2];2161 my $chunk_name = ${ $this_warp }[6];2162 1942 } 2163 1943 else { 2164 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected$this_comment\n";1944 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n"; 2165 1945 } 2166 1946 } 2167 @{ $warps } = @keep_warps; 2168 my $nwarps = ($#{ $warps } + 1); 2169 2170 ################################ 2171 #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs 2172 #Conversely, less than 3 warps might result in dirty diffs 2173 ################################ 2174 if($nwarps >= 3) { 2175 # We are attempting to do the missing diffs, so reverse the list of retained warps. 2176 # Good objects with all visits will be skipped due to the duplicate check. 2177 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 2178 @{ $warps } = reverse @keep_warps; 2179 2180 # Exclude the last entry if we do not have an even number of warps. 2181 if (($#{ $warps } + 1) % 2 != 0) { 2182 print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; 2183 if ($#{ $warps} + 1 == 1) { 2184 print STDERR ": I can do no diffs with only one exposure.\n"; 2185 next; 2186 } 2187 else { 2188 my $rejected_warp = pop @{ $warps }; 2189 my $rejected_exp_id = ${ $rejected_warp }[0]; 2190 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 2191 } 1947 # We are attempting to do the missing diffs, so reverse the list of retained warps. 1948 # Good objects with all visits will be skipped due to the duplicate check. 1949 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 1950 @{ $warps } = reverse @keep_warps; 1951 1952 # Exclude the last entry if we do not have an even number of warps. 1953 if (($#{ $warps } + 1) % 2 != 0) { 1954 print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; 1955 if ($#{ $warps} + 1 == 1) { 1956 print STDERR ": I can do no diffs with only one exposure.\n"; 1957 next; 2192 1958 } 1959 else { 1960 my $rejected_warp = pop @{ $warps }; 1961 my $rejected_exp_id = ${ $rejected_warp }[0]; 1962 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 1963 } 1964 } 2193 1965 2194 2195 # The array is sorted in pairs of input/template.2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 1966 while ($#{ $warps } > -1) { 1967 # In this mode, the array is sorted in pairs of template/input, counter to the standard queue. 1968 my $template_warp = shift @{ $warps }; 1969 my $input_warp = shift @{ $warps }; 1970 1971 my $input_exp_id = ${ $input_warp }[0]; 1972 my $input_comment = ${ $input_warp }[3]; 1973 1974 my $template_exp_id = ${ $template_warp }[0]; 1975 my $template_comment = ${ $template_warp }[3]; 1976 1977 my $input_warp_id = ${ $input_warp }[1]; 1978 my $template_warp_id = ${ $template_warp }[1]; 1979 1980 my $input_warp_state = ${ $input_warp }[4]; 1981 my $template_warp_state = ${ $template_warp }[4]; 1982 1983 my $input_warp_camQuality = ${ $input_warp }[5]; 1984 my $template_warp_camQuality = ${ $template_warp }[5]; 1985 1986 $Npotential++; 1987 1988 unless (defined($input_warp_id) && defined($template_warp_id) && 2217 1989 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 2218 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 2219 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 2220 # This should now never be reached. 2221 # CZW: Trigger backup plan here? Or simply set up framework? 2222 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 2223 $Npotential--; 2224 } 2225 next; 2226 } 2227 2228 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 2229 $Nqueued++; 2230 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2231 next; 2232 } 2233 else { 2234 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2235 } 2236 2237 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 2238 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 2239 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 2240 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 2241 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 2242 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 2243 if (defined($reduction)) { 2244 $cmd .= " -set_reduction $reduction "; 2245 } 2246 2247 if (defined($pretend)) { 2248 $cmd .= ' -pretend '; 2249 } 2250 if ($debug == 1) { 2251 $cmd .= ' -pretend '; 2252 print STDERR "desp_diff_queue: $cmd\n"; 2253 print STDERR " $input_warp_id $template_warp_id\n"; 2254 } 2255 2256 if (($debug == 0)&&(!defined($pretend))) { 2257 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2258 run ( command => $cmd, verbose => $verbose ); 2259 unless ($success) { 2260 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2261 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 2262 } 2263 $Nqueued++; 2264 } 2265 } 2266 2267 } else { 2268 #Consider the case of on-the-fly dirty desperate diffs 2269 #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old 2270 2271 # Exclude any warps that are not stored in the more general comment_hash. 2272 my @keep_warps = (); 2273 foreach my $this_warp (@{ $warpsBQ }) { 2274 my $this_comment = ${ $this_warp }[3]; 2275 my $this_exp_id = ${ $this_warp }[0]; 2276 if ((exists($comment_hash{$this_comment}))&& 2277 ($comment_hash{$this_comment} == $this_exp_id)) { 2278 push @keep_warps, $this_warp; 2279 my $this_date = ${ $this_warp }[2]; 2280 my $chunk_name = ${ $this_warp }[6]; 2281 } 2282 else { 2283 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; 2284 } 1990 print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 1991 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 1992 # This should now never be reached. 1993 # CZW: Trigger backup plan here? Or simply set up framework? 1994 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 1995 $Npotential--; 1996 } 1997 next; 2285 1998 } 2286 @{ $warpsBQ } = @keep_warps; 2287 2288 if ($nwarps >= 2) { 2289 #one regular diff has already been made. So, need only one more. Strip a good entry from the array 2290 my $nrem = 0; 2291 my @keep_warps = (); 2292 foreach my $this_warp (@{ $warpsBQ }) { 2293 my $this_comment = ${ $this_warp }[3]; 2294 my $this_exp_id = ${ $this_warp }[0]; 2295 my $this_fwhm = ${ $this_warp }[7]; 2296 if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { 2297 $nrem++; 2298 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; 2299 } else { 2300 push @keep_warps, $this_warp; 2301 } 2302 } 2303 @{ $warpsBQ } = @keep_warps; 1999 2000 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 2001 $Nqueued++; 2002 print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2003 next; 2304 2004 } 2305 2306 # We are attempting to do the missing diffs, so reverse the list of retained warps. 2307 # Good objects with all visits will be skipped due to the duplicate check. 2308 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 2309 @{ $warpsBQ } = reverse @{ $warpsBQ }; 2310 2311 # Exclude the last entry if we do not have an even number of warps. 2312 if (($#{ $warpsBQ } + 1) % 2 != 0) { 2313 print STDERR "desp_diff_queue: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; 2314 my $rejected_warp = pop @{ $warpsBQ }; 2315 my $rejected_exp_id = ${ $rejected_warp }[0]; 2316 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 2317 } 2318 my $nwarpsBQ = ($#{$warpsBQ} + 1); 2319 2320 2321 while ($#{ $warpsBQ } > -1) { 2322 # The array is sorted in pairs of input/template. 2323 my $template_warp = shift @{ $warpsBQ }; 2324 my $input_warp = shift @{ $warpsBQ }; 2325 2326 my $input_exp_id = ${ $input_warp }[0]; 2327 my $input_comment = ${ $input_warp }[3]; 2328 2329 my $template_exp_id = ${ $template_warp }[0]; 2330 my $template_comment = ${ $template_warp }[3]; 2331 2332 my $input_warp_id = ${ $input_warp }[1]; 2333 my $template_warp_id = ${ $template_warp }[1]; 2334 2335 my $input_warp_state = ${ $input_warp }[4]; 2336 my $template_warp_state = ${ $template_warp }[4]; 2337 2338 my $input_warp_camQuality = ${ $input_warp }[5]; 2339 my $template_warp_camQuality = ${ $template_warp }[5]; 2340 2341 $Npotential++; 2342 2343 unless (defined($input_warp_id) && defined($template_warp_id) && 2344 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 2345 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 2346 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 2347 # This should now never be reached. 2348 # CZW: Trigger backup plan here? Or simply set up framework? 2349 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 2350 $Npotential--; 2351 } 2352 next; 2353 } 2354 2355 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 2356 $Nqueued++; 2357 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2358 next; 2359 } 2360 else { 2361 print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2362 } 2363 2364 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 2365 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 2366 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 2367 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 2368 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 2369 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 2370 if (defined($reduction)) { 2371 $cmd .= " -set_reduction $reduction "; 2372 } 2373 2374 if (defined($pretend)) { 2375 $cmd .= ' -pretend '; 2376 } 2377 if ($debug == 1) { 2378 $cmd .= ' -pretend '; 2379 print STDERR "desp_diff_queue: $cmd\n"; 2380 print STDERR " $input_warp_id $template_warp_id\n"; 2381 } 2382 2383 if (($debug == 0)&&(!defined($pretend))) { 2384 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2385 run ( command => $cmd, verbose => $verbose ); 2386 unless ($success) { 2387 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2388 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 2389 } 2390 $Nqueued++; 2391 } 2005 else { 2006 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2392 2007 } 2393 2008 2009 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 2010 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 2011 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 2012 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 2013 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 2014 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 2015 # $cmd .= " -pretend "; 2016 if (defined($reduction)) { 2017 $cmd .= " -set_reduction $reduction "; 2018 } 2019 2020 if (defined($pretend)) { 2021 $cmd .= ' -pretend '; 2022 } 2023 if ($debug == 1) { 2024 $cmd .= ' -pretend '; 2025 print STDERR "desp_diff_queue: $cmd\n"; 2026 print STDERR " $input_warp_id $template_warp_id\n"; 2027 } 2394 2028 2395 2029 if (($debug == 0)&&(!defined($pretend))) { 2030 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2031 run ( command => $cmd, verbose => $verbose ); 2032 unless ($success) { 2033 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2034 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 2035 } 2036 $Nqueued++; 2037 } 2396 2038 } 2397 2039 } … … 2399 2041 $metadata_out{nsDiffPotential} += $Npotential; 2400 2042 $metadata_out{nsDiffQueued} += $Nqueued; 2043 # if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { 2044 # $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; 2045 # } 2046 2401 2047 } 2402 2048
Note:
See TracChangeset
for help on using the changeset viewer.
