Changeset 42154 for trunk/ippScripts/scripts/nightly_science.pl
- Timestamp:
- Apr 4, 2022, 2:56:53 PM (4 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/nightly_science.pl (modified) (18 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/nightly_science.pl
r41994 r42154 142 142 my %unique_filter_hash = (); 143 143 144 #set up the camera config 145 my $ipprcam = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up camera", $PS_EXIT_CONFIG_ERROR ); # IPP camera configuration 146 144 147 # Grab the configuration data. 145 148 my $conf_cmd = "$ppConfigDump -camera $camera -dump-recipe NIGHTLY_SCIENCE -"; … … 1391 1394 foreach my $target (sort (keys %science_config)) { 1392 1395 if ($science_config{$target}{DIFFABLE} == 1) { 1396 1397 my $config_reduction = "DEFAULT"; 1398 if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};} 1399 print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n"; 1400 1401 my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use 1402 &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; 1403 1404 my $recipeData; 1405 { 1406 # Get the PSASTRO recipe 1407 my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; 1408 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1409 run(command => $command); 1410 unless ($success) { 1411 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1412 &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR); 1413 } 1414 $recipeData = $mdcParser->parse(join "", @$stdout_buf) or 1415 &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR); 1416 } 1417 1418 my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); 1419 print STDERR "execute_diffs: Max FWHM= $maxFWHM for $target\n"; 1420 1393 1421 foreach my $filter (@filter_list) { 1394 1422 $Npotential ++; … … 1413 1441 print STDERR "execute_diffs: Target $target in filter $filter on $date has exposures and will be queued.\n"; 1414 1442 } 1415 diff_queue($date,$target,$filter,$ pretend);1443 diff_queue($date,$target,$filter,$maxFWHM,$pretend); 1416 1444 1417 1445 #Queue up desperate diffs if the conditions are right for them 1418 1446 if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { 1419 desperate_diff_singles($date,$target,$filter,$ pretend);1447 desperate_diff_singles($date,$target,$filter,$maxFWHM,$pretend); 1420 1448 } 1421 1449 … … 1459 1487 if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') { 1460 1488 foreach my $target (sort (keys %science_config)) { 1489 1490 my $config_reduction = "DEFAULT"; 1491 if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};} 1492 print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n"; 1493 1494 my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use 1495 &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; 1496 1497 my $recipeData; 1498 { 1499 # Get the PSASTRO recipe 1500 my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; 1501 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1502 run(command => $command); 1503 unless ($success) { 1504 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1505 &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR); 1506 } 1507 $recipeData = $mdcParser->parse(join "", @$stdout_buf) or 1508 &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR); 1509 } 1510 1511 my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); 1512 1461 1513 # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now. 1462 1514 if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { 1463 1515 foreach my $filter (@filter_list) { 1464 desperate_diff_queue($date,$target,$filter,$ pretend);1516 desperate_diff_queue($date,$target,$filter,$maxFWHM,$pretend); 1465 1517 } 1466 1518 } … … 1488 1540 my $target = shift; 1489 1541 my $filter = shift; 1542 my $maxFWHM = shift; 1490 1543 my $pretend = shift; 1491 1544 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); … … 1507 1560 $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; 1508 1561 $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; 1562 $input_sth .= " AND camProcessedExp.fwhm_major <= $maxFWHM "; 1509 1563 $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs "; 1510 1564 … … 1639 1693 my $target = shift; 1640 1694 my $filter = shift; 1695 my $maxFWHM = shift; 1641 1696 my $pretend = shift; 1642 1697 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); … … 1694 1749 my $this_chunk = shift @{ $object_row }; 1695 1750 1696 my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM ";1751 my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)),camProcessedExp.fwhm_major FROM "; 1697 1752 $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; 1698 1753 $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; … … 1700 1755 1701 1756 my $warps = $db->selectall_arrayref( $input_sth ); 1757 my $warpsBQ = $db->selectall_arrayref( $input_sth );; 1702 1758 1703 1759 # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. 1704 1760 my %comment_hash = (); 1761 my %comment_hash_good = (); 1705 1762 foreach my $this_warp (@{ $warps }) { 1706 1763 my $this_comment = ${ $this_warp }[3]; … … 1708 1765 my $this_quality = ${ $this_warp }[5]; 1709 1766 my $this_state = ${ $this_warp }[4]; 1710 if (($this_quality != 0) || ($this_state eq 'drop') ) { 1711 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; 1767 my $this_fwhm = ${ $this_warp }[7]; 1768 if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { 1769 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; 1712 1770 } 1713 1771 else { 1772 $comment_hash_good{$this_comment} = $this_exp_id; 1773 } 1774 1775 #also save the entries which are ok, but not best quality 1776 if (($this_quality == 0) && ($this_state ne 'drop') ) { 1714 1777 $comment_hash{$this_comment} = $this_exp_id; 1715 1778 } 1716 1779 } 1717 # Exclude any warps that are not stored in the comment_hash. 1780 1781 # Exclude any warps that are not stored in the comment_hash_good. 1718 1782 my @keep_warps = (); 1719 1783 foreach my $this_warp (@{ $warps }) { 1720 1784 my $this_comment = ${ $this_warp }[3]; 1721 1785 my $this_exp_id = ${ $this_warp }[0]; 1722 if ((exists($comment_hash {$this_comment}))&&1723 ($comment_hash {$this_comment} == $this_exp_id)) {1786 if ((exists($comment_hash_good{$this_comment}))&& 1787 ($comment_hash_good{$this_comment} == $this_exp_id)) { 1724 1788 push @keep_warps, $this_warp; 1725 1789 $this_date = ${ $this_warp }[2]; … … 1731 1795 } 1732 1796 @{ $warps } = @keep_warps; 1733 1797 my $nwarps = ($#{ $warps } + 1); 1798 1734 1799 #find the time of the most recent exposure in that chunk 1735 1800 my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' "; … … 1740 1805 1741 1806 #compute and store some stats for potential on-the-fly desperate diffs 1742 my $nwarps = ($#{ $warps } + 1);1743 1807 my $timediff = 0; 1744 1808 my $now=DateTime->now; … … 1752 1816 } 1753 1817 } 1754 1755 #Consider the special case of on-the-fly desperate diffs 1756 #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old 1757 if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { 1758 print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; 1759 next; 1760 } 1761 if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { 1762 next; 1763 } 1764 1765 # We are attempting to do the missing diffs, so reverse the list of retained warps. 1766 # Good objects with all visits will be skipped due to the duplicate check. 1767 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 1768 @{ $warps } = reverse @keep_warps; 1769 1770 # Exclude the last entry if we do not have an even number of warps. 1771 if (($#{ $warps } + 1) % 2 != 0) { 1772 print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; 1773 my $rejected_warp = pop @{ $warps }; 1774 my $rejected_exp_id = ${ $rejected_warp }[0]; 1775 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 1776 } 1818 1819 ################################ 1820 #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs 1821 #Conversely, less than 3 good quality warps might result in dirty diffs 1822 ################################ 1823 if($nwarps >= 3) { 1824 #Consider the special case of on-the-fly desperate diffs 1825 #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old 1826 if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { 1827 print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; 1828 next; 1829 } 1830 if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { 1831 next; 1832 } 1833 1834 # We are attempting to do the missing diffs, so reverse the list of retained warps. 1835 # Good objects with all visits will be skipped due to the duplicate check. 1836 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 1837 @{ $warps } = reverse @keep_warps; 1838 1839 # Exclude the last entry if we do not have an even number of warps. 1840 if (($#{ $warps } + 1) % 2 != 0) { 1841 print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; 1842 my $rejected_warp = pop @{ $warps }; 1843 my $rejected_exp_id = ${ $rejected_warp }[0]; 1844 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 1845 } 1777 1846 1778 while ($#{ $warps } > -1) {1779 # The array is sorted in pairs of input/template.1780 my $template_warp = shift @{ $warps };1781 my $input_warp = shift @{ $warps };1847 while ($#{ $warps } > -1) { 1848 # The array is sorted in pairs of input/template. 1849 my $template_warp = shift @{ $warps }; 1850 my $input_warp = shift @{ $warps }; 1782 1851 1783 my $input_exp_id = ${ $input_warp }[0]; 1784 my $input_comment = ${ $input_warp }[3]; 1785 1786 my $template_exp_id = ${ $template_warp }[0]; 1787 my $template_comment = ${ $template_warp }[3]; 1788 1789 my $input_warp_id = ${ $input_warp }[1]; 1790 my $template_warp_id = ${ $template_warp }[1]; 1791 1792 my $input_warp_state = ${ $input_warp }[4]; 1793 my $template_warp_state = ${ $template_warp }[4]; 1794 1795 my $input_warp_camQuality = ${ $input_warp }[5]; 1796 my $template_warp_camQuality = ${ $template_warp }[5]; 1797 1798 $Npotential++; 1799 1800 unless (defined($input_warp_id) && defined($template_warp_id) && 1801 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 1802 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 1803 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 1804 # This should now never be reached. 1805 # CZW: Trigger backup plan here? Or simply set up framework? 1806 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 1807 $Npotential--; 1852 my $input_exp_id = ${ $input_warp }[0]; 1853 my $input_comment = ${ $input_warp }[3]; 1854 1855 my $template_exp_id = ${ $template_warp }[0]; 1856 my $template_comment = ${ $template_warp }[3]; 1857 1858 my $input_warp_id = ${ $input_warp }[1]; 1859 my $template_warp_id = ${ $template_warp }[1]; 1860 1861 my $input_warp_state = ${ $input_warp }[4]; 1862 my $template_warp_state = ${ $template_warp }[4]; 1863 1864 my $input_warp_camQuality = ${ $input_warp }[5]; 1865 my $template_warp_camQuality = ${ $template_warp }[5]; 1866 1867 $Npotential++; 1868 1869 unless (defined($input_warp_id) && defined($template_warp_id) && 1870 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 1871 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 1872 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 1873 # This should now never be reached. 1874 # CZW: Trigger backup plan here? Or simply set up framework? 1875 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 1876 $Npotential--; 1877 } 1878 next; 1879 } 1880 1881 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 1882 $Nqueued++; 1883 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 1884 next; 1885 } 1886 else { 1887 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 1888 } 1889 1890 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 1891 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 1892 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 1893 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 1894 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 1895 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 1896 if (defined($reduction)) { 1897 $cmd .= " -set_reduction $reduction "; 1898 } 1899 1900 if (defined($pretend)) { 1901 $cmd .= ' -pretend '; 1902 } 1903 if ($debug == 1) { 1904 $cmd .= ' -pretend '; 1905 print STDERR "desp_diff_singles: $cmd\n"; 1906 print STDERR " $input_warp_id $template_warp_id\n"; 1907 } 1908 1909 if (($debug == 0)&&(!defined($pretend))) { 1910 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1911 run ( command => $cmd, verbose => $verbose ); 1912 unless ($success) { 1913 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1914 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 1915 } 1916 $Nqueued++; 1917 } 1918 } 1919 1920 } else { 1921 #Consider the case of on-the-fly dirty desperate diffs 1922 #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old 1923 1924 # Exclude any warps that are not stored in the more general comment_hash. 1925 my @keep_warps = (); 1926 foreach my $this_warp (@{ $warpsBQ }) { 1927 my $this_comment = ${ $this_warp }[3]; 1928 my $this_exp_id = ${ $this_warp }[0]; 1929 if ((exists($comment_hash{$this_comment}))&& 1930 ($comment_hash{$this_comment} == $this_exp_id)) { 1931 push @keep_warps, $this_warp; 1932 $this_date = ${ $this_warp }[2]; 1933 $chunk_name = ${ $this_warp }[6]; 1934 } 1935 else { 1936 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; 1937 } 1938 } 1939 @{ $warpsBQ } = @keep_warps; 1940 1941 if ($nwarps >= 2) { 1942 #one regular diff has already been made. So, need only one more. Strip a good entry from the array 1943 my $nrem = 0; 1944 my @keep_warps = (); 1945 foreach my $this_warp (@{ $warpsBQ }) { 1946 my $this_comment = ${ $this_warp }[3]; 1947 my $this_exp_id = ${ $this_warp }[0]; 1948 my $this_fwhm = ${ $this_warp }[7]; 1949 if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { 1950 $nrem++; 1951 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; 1952 } else { 1953 push @keep_warps, $this_warp; 1954 } 1808 1955 } 1809 next;1956 @{ $warpsBQ } = @keep_warps; 1810 1957 } 1811 1958 1812 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 1813 $Nqueued++; 1814 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 1815 next; 1959 # We are attempting to do the missing diffs, so reverse the list of retained warps. 1960 # Good objects with all visits will be skipped due to the duplicate check. 1961 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 1962 @{ $warpsBQ } = reverse @{ $warpsBQ }; 1963 1964 # Exclude the last entry if we do not have an even number of warps. 1965 if (($#{ $warpsBQ } + 1) % 2 != 0) { 1966 print STDERR "desp_diff_singles: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; 1967 my $rejected_warp = pop @{ $warpsBQ }; 1968 my $rejected_exp_id = ${ $rejected_warp }[0]; 1969 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 1816 1970 } 1817 else { 1818 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 1971 my $nwarpsBQ = ($#{$warpsBQ} + 1); 1972 1973 1974 if (($nwarpsBQ >= 2) && ($timediff < $desdiffdt)) { 1975 print STDERR "desp_diff_singles: There are potential dirty desperate diffs to be done, but the time criterium is not met.\n"; 1976 next; 1977 } 1978 if (($nwarpsBQ <= 1) || ($timediff < $desdiffdt)) { 1979 next; 1980 } 1981 1982 while ($#{ $warpsBQ } > -1) { 1983 # The array is sorted in pairs of input/template. 1984 my $template_warp = shift @{ $warpsBQ }; 1985 my $input_warp = shift @{ $warpsBQ }; 1986 1987 my $input_exp_id = ${ $input_warp }[0]; 1988 my $input_comment = ${ $input_warp }[3]; 1989 1990 my $template_exp_id = ${ $template_warp }[0]; 1991 my $template_comment = ${ $template_warp }[3]; 1992 1993 my $input_warp_id = ${ $input_warp }[1]; 1994 my $template_warp_id = ${ $template_warp }[1]; 1995 1996 my $input_warp_state = ${ $input_warp }[4]; 1997 my $template_warp_state = ${ $template_warp }[4]; 1998 1999 my $input_warp_camQuality = ${ $input_warp }[5]; 2000 my $template_warp_camQuality = ${ $template_warp }[5]; 2001 2002 $Npotential++; 2003 2004 unless (defined($input_warp_id) && defined($template_warp_id) && 2005 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 2006 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 2007 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 2008 # This should now never be reached. 2009 # CZW: Trigger backup plan here? Or simply set up framework? 2010 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 2011 $Npotential--; 2012 } 2013 next; 2014 } 2015 2016 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 2017 $Nqueued++; 2018 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2019 next; 2020 } 2021 else { 2022 print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2023 } 2024 2025 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 2026 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 2027 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 2028 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 2029 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 2030 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 2031 if (defined($reduction)) { 2032 $cmd .= " -set_reduction $reduction "; 2033 } 2034 2035 if (defined($pretend)) { 2036 $cmd .= ' -pretend '; 2037 } 2038 if ($debug == 1) { 2039 $cmd .= ' -pretend '; 2040 print STDERR "desp_diff_singles: $cmd\n"; 2041 print STDERR " $input_warp_id $template_warp_id\n"; 2042 } 2043 2044 if (($debug == 0)&&(!defined($pretend))) { 2045 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2046 run ( command => $cmd, verbose => $verbose ); 2047 unless ($success) { 2048 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2049 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 2050 } 2051 $Nqueued++; 2052 } 1819 2053 } 1820 2054 1821 my $cmd = "$difftool -dbname $dbname -definewarpwarp ";1822 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 ";1823 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.1824 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures1825 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group ";1826 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";1827 if (defined($reduction)) {1828 $cmd .= " -set_reduction $reduction ";1829 }1830 1831 if (defined($pretend)) {1832 $cmd .= ' -pretend ';1833 }1834 if ($debug == 1) {1835 $cmd .= ' -pretend ';1836 print STDERR "desp_diff_singles: $cmd\n";1837 print STDERR " $input_warp_id $template_warp_id\n";1838 }1839 2055 1840 if (($debug == 0)&&(!defined($pretend))) { 1841 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1842 run ( command => $cmd, verbose => $verbose ); 1843 unless ($success) { 1844 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1845 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 1846 } 1847 $Nqueued++; 1848 } 2056 1849 2057 } 1850 2058 } … … 1858 2066 my $target = shift; 1859 2067 my $filter = shift; 2068 my $maxFWHM = shift; 1860 2069 my $pretend = shift; 1861 2070 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); … … 1911 2120 my $this_chunk = shift @{ $object_row }; 1912 2121 1913 my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM ";2122 my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,camProcessedExp.fwhm_major FROM "; 1914 2123 $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; 1915 2124 $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; … … 1917 2126 1918 2127 my $warps = $db->selectall_arrayref( $input_sth ); 2128 my $warpsBQ = $db->selectall_arrayref( $input_sth );; 1919 2129 1920 2130 # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. 1921 2131 my %comment_hash = (); 2132 my %comment_hash_good = (); 1922 2133 foreach my $this_warp (@{ $warps }) { 1923 2134 my $this_comment = ${ $this_warp }[3]; … … 1925 2136 my $this_quality = ${ $this_warp }[5]; 1926 2137 my $this_state = ${ $this_warp }[4]; 1927 if (($this_quality != 0) || ($this_state eq 'drop') ) { 1928 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; 2138 my $this_fwhm = ${ $this_warp }[7]; 2139 if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { 2140 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; 1929 2141 } 1930 2142 else { 2143 $comment_hash_good{$this_comment} = $this_exp_id; 2144 } 2145 2146 #also save the entries which are ok, but not best quality 2147 if (($this_quality == 0) && ($this_state ne 'drop') ) { 1931 2148 $comment_hash{$this_comment} = $this_exp_id; 1932 2149 } 1933 2150 } 1934 # Exclude any warps that are not stored in the comment_hash. 2151 2152 # Exclude any warps that are not stored in the comment_hash_good. 1935 2153 my @keep_warps = (); 1936 2154 foreach my $this_warp (@{ $warps }) { 1937 2155 my $this_comment = ${ $this_warp }[3]; 1938 2156 my $this_exp_id = ${ $this_warp }[0]; 1939 if ((exists($comment_hash {$this_comment}))&&1940 ($comment_hash {$this_comment} == $this_exp_id)) {2157 if ((exists($comment_hash_good{$this_comment}))&& 2158 ($comment_hash_good{$this_comment} == $this_exp_id)) { 1941 2159 push @keep_warps, $this_warp; 2160 my $this_date = ${ $this_warp }[2]; 2161 my $chunk_name = ${ $this_warp }[6]; 1942 2162 } 1943 2163 else { 1944 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string$this_comment\n";2164 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; 1945 2165 } 1946 2166 } 1947 # We are attempting to do the missing diffs, so reverse the list of retained warps. 1948 # Good objects with all visits will be skipped due to the duplicate check. 1949 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 1950 @{ $warps } = reverse @keep_warps; 1951 1952 # Exclude the last entry if we do not have an even number of warps. 1953 if (($#{ $warps } + 1) % 2 != 0) { 1954 print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; 1955 if ($#{ $warps} + 1 == 1) { 1956 print STDERR ": I can do no diffs with only one exposure.\n"; 1957 next; 2167 @{ $warps } = @keep_warps; 2168 my $nwarps = ($#{ $warps } + 1); 2169 2170 ################################ 2171 #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs 2172 #Conversely, less than 3 warps might result in dirty diffs 2173 ################################ 2174 if($nwarps >= 3) { 2175 # We are attempting to do the missing diffs, so reverse the list of retained warps. 2176 # Good objects with all visits will be skipped due to the duplicate check. 2177 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 2178 @{ $warps } = reverse @keep_warps; 2179 2180 # Exclude the last entry if we do not have an even number of warps. 2181 if (($#{ $warps } + 1) % 2 != 0) { 2182 print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; 2183 if ($#{ $warps} + 1 == 1) { 2184 print STDERR ": I can do no diffs with only one exposure.\n"; 2185 next; 2186 } 2187 else { 2188 my $rejected_warp = pop @{ $warps }; 2189 my $rejected_exp_id = ${ $rejected_warp }[0]; 2190 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 2191 } 1958 2192 } 1959 else { 1960 my $rejected_warp = pop @{ $warps }; 1961 my $rejected_exp_id = ${ $rejected_warp }[0]; 1962 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 2193 2194 while ($#{ $warps } > -1) { 2195 # The array is sorted in pairs of input/template. 2196 my $template_warp = shift @{ $warps }; 2197 my $input_warp = shift @{ $warps }; 2198 2199 my $input_exp_id = ${ $input_warp }[0]; 2200 my $input_comment = ${ $input_warp }[3]; 2201 2202 my $template_exp_id = ${ $template_warp }[0]; 2203 my $template_comment = ${ $template_warp }[3]; 2204 2205 my $input_warp_id = ${ $input_warp }[1]; 2206 my $template_warp_id = ${ $template_warp }[1]; 2207 2208 my $input_warp_state = ${ $input_warp }[4]; 2209 my $template_warp_state = ${ $template_warp }[4]; 2210 2211 my $input_warp_camQuality = ${ $input_warp }[5]; 2212 my $template_warp_camQuality = ${ $template_warp }[5]; 2213 2214 $Npotential++; 2215 2216 unless (defined($input_warp_id) && defined($template_warp_id) && 2217 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 2218 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 2219 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 2220 # This should now never be reached. 2221 # CZW: Trigger backup plan here? Or simply set up framework? 2222 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 2223 $Npotential--; 2224 } 2225 next; 2226 } 2227 2228 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 2229 $Nqueued++; 2230 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2231 next; 2232 } 2233 else { 2234 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2235 } 2236 2237 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 2238 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 2239 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 2240 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 2241 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 2242 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 2243 if (defined($reduction)) { 2244 $cmd .= " -set_reduction $reduction "; 2245 } 2246 2247 if (defined($pretend)) { 2248 $cmd .= ' -pretend '; 2249 } 2250 if ($debug == 1) { 2251 $cmd .= ' -pretend '; 2252 print STDERR "desp_diff_queue: $cmd\n"; 2253 print STDERR " $input_warp_id $template_warp_id\n"; 2254 } 2255 2256 if (($debug == 0)&&(!defined($pretend))) { 2257 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2258 run ( command => $cmd, verbose => $verbose ); 2259 unless ($success) { 2260 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2261 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 2262 } 2263 $Nqueued++; 2264 } 2265 } 2266 2267 } else { 2268 #Consider the case of on-the-fly dirty desperate diffs 2269 #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old 2270 2271 # Exclude any warps that are not stored in the more general comment_hash. 2272 my @keep_warps = (); 2273 foreach my $this_warp (@{ $warpsBQ }) { 2274 my $this_comment = ${ $this_warp }[3]; 2275 my $this_exp_id = ${ $this_warp }[0]; 2276 if ((exists($comment_hash{$this_comment}))&& 2277 ($comment_hash{$this_comment} == $this_exp_id)) { 2278 push @keep_warps, $this_warp; 2279 my $this_date = ${ $this_warp }[2]; 2280 my $chunk_name = ${ $this_warp }[6]; 2281 } 2282 else { 2283 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; 2284 } 1963 2285 } 1964 } 2286 @{ $warpsBQ } = @keep_warps; 2287 2288 if ($nwarps >= 2) { 2289 #one regular diff has already been made. So, need only one more. Strip a good entry from the array 2290 my $nrem = 0; 2291 my @keep_warps = (); 2292 foreach my $this_warp (@{ $warpsBQ }) { 2293 my $this_comment = ${ $this_warp }[3]; 2294 my $this_exp_id = ${ $this_warp }[0]; 2295 my $this_fwhm = ${ $this_warp }[7]; 2296 if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { 2297 $nrem++; 2298 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; 2299 } else { 2300 push @keep_warps, $this_warp; 2301 } 2302 } 2303 @{ $warpsBQ } = @keep_warps; 2304 } 2305 2306 # We are attempting to do the missing diffs, so reverse the list of retained warps. 2307 # Good objects with all visits will be skipped due to the duplicate check. 2308 # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. 2309 @{ $warpsBQ } = reverse @{ $warpsBQ }; 2310 2311 # Exclude the last entry if we do not have an even number of warps. 2312 if (($#{ $warpsBQ } + 1) % 2 != 0) { 2313 print STDERR "desp_diff_queue: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; 2314 my $rejected_warp = pop @{ $warpsBQ }; 2315 my $rejected_exp_id = ${ $rejected_warp }[0]; 2316 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; 2317 } 2318 my $nwarpsBQ = ($#{$warpsBQ} + 1); 1965 2319 1966 while ($#{ $warps } > -1) { 1967 # In this mode, the array is sorted in pairs of template/input, counter to the standard queue. 1968 my $template_warp = shift @{ $warps }; 1969 my $input_warp = shift @{ $warps }; 1970 1971 my $input_exp_id = ${ $input_warp }[0]; 1972 my $input_comment = ${ $input_warp }[3]; 1973 1974 my $template_exp_id = ${ $template_warp }[0]; 1975 my $template_comment = ${ $template_warp }[3]; 1976 1977 my $input_warp_id = ${ $input_warp }[1]; 1978 my $template_warp_id = ${ $template_warp }[1]; 1979 1980 my $input_warp_state = ${ $input_warp }[4]; 1981 my $template_warp_state = ${ $template_warp }[4]; 1982 1983 my $input_warp_camQuality = ${ $input_warp }[5]; 1984 my $template_warp_camQuality = ${ $template_warp }[5]; 1985 1986 $Npotential++; 1987 1988 unless (defined($input_warp_id) && defined($template_warp_id) && 2320 2321 while ($#{ $warpsBQ } > -1) { 2322 # The array is sorted in pairs of input/template. 2323 my $template_warp = shift @{ $warpsBQ }; 2324 my $input_warp = shift @{ $warpsBQ }; 2325 2326 my $input_exp_id = ${ $input_warp }[0]; 2327 my $input_comment = ${ $input_warp }[3]; 2328 2329 my $template_exp_id = ${ $template_warp }[0]; 2330 my $template_comment = ${ $template_warp }[3]; 2331 2332 my $input_warp_id = ${ $input_warp }[1]; 2333 my $template_warp_id = ${ $template_warp }[1]; 2334 2335 my $input_warp_state = ${ $input_warp }[4]; 2336 my $template_warp_state = ${ $template_warp }[4]; 2337 2338 my $input_warp_camQuality = ${ $input_warp }[5]; 2339 my $template_warp_camQuality = ${ $template_warp }[5]; 2340 2341 $Npotential++; 2342 2343 unless (defined($input_warp_id) && defined($template_warp_id) && 1989 2344 ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { 1990 print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 1991 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 1992 # This should now never be reached. 1993 # CZW: Trigger backup plan here? Or simply set up framework? 1994 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 1995 $Npotential--; 1996 } 1997 next; 1998 } 1999 2000 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 2001 $Nqueued++; 2002 print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2003 next; 2004 } 2005 else { 2006 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2345 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; 2346 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { 2347 # This should now never be reached. 2348 # CZW: Trigger backup plan here? Or simply set up framework? 2349 print STDERR " ...but this is due to a camera stage astrometry quality\n"; 2350 $Npotential--; 2351 } 2352 next; 2353 } 2354 2355 if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { 2356 $Nqueued++; 2357 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2358 next; 2359 } 2360 else { 2361 print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; 2362 } 2363 2364 my $cmd = "$difftool -dbname $dbname -definewarpwarp "; 2365 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; 2366 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. 2367 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures 2368 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; 2369 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; 2370 if (defined($reduction)) { 2371 $cmd .= " -set_reduction $reduction "; 2372 } 2373 2374 if (defined($pretend)) { 2375 $cmd .= ' -pretend '; 2376 } 2377 if ($debug == 1) { 2378 $cmd .= ' -pretend '; 2379 print STDERR "desp_diff_queue: $cmd\n"; 2380 print STDERR " $input_warp_id $template_warp_id\n"; 2381 } 2382 2383 if (($debug == 0)&&(!defined($pretend))) { 2384 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2385 run ( command => $cmd, verbose => $verbose ); 2386 unless ($success) { 2387 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2388 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 2389 } 2390 $Nqueued++; 2391 } 2007 2392 } 2008 2393 2009 my $cmd = "$difftool -dbname $dbname -definewarpwarp ";2010 $cmd .= "-input_label $label -template_label $label -good_frac 0.1 ";2011 $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.2012 $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures2013 $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group ";2014 $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";2015 # $cmd .= " -pretend ";2016 if (defined($reduction)) {2017 $cmd .= " -set_reduction $reduction ";2018 }2019 2020 if (defined($pretend)) {2021 $cmd .= ' -pretend ';2022 }2023 if ($debug == 1) {2024 $cmd .= ' -pretend ';2025 print STDERR "desp_diff_queue: $cmd\n";2026 print STDERR " $input_warp_id $template_warp_id\n";2027 }2028 2394 2029 if (($debug == 0)&&(!defined($pretend))) { 2030 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 2031 run ( command => $cmd, verbose => $verbose ); 2032 unless ($success) { 2033 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 2034 &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 2035 } 2036 $Nqueued++; 2037 } 2395 2038 2396 } 2039 2397 } … … 2041 2399 $metadata_out{nsDiffPotential} += $Npotential; 2042 2400 $metadata_out{nsDiffQueued} += $Nqueued; 2043 # if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {2044 # $metadata_out{nsDiffState} = 'FINISHED_DIFFS';2045 # }2046 2047 2401 } 2048 2402
Note:
See TracChangeset
for help on using the changeset viewer.
