IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 41466


Ignore:
Timestamp:
Jan 6, 2021, 4:29:01 PM (6 years ago)
Author:
tdeboer
Message:

merging fix for nsDiffState to nightly_science.pl

File:
1 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-ps2-20190404/ippScripts/scripts/nightly_science.pl

    r41227 r41466  
    14621462            if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) {
    14631463                foreach my $filter (@filter_list) {
    1464                     # This one needs to return a state to see if we need to wait on stacking before checking again.
    1465                     $metadata_out{nsDiffState} = desperate_diff_queue($date,$target,$filter,$pretend);
     1464                    desperate_diff_queue($date,$target,$filter,$pretend);
    14661465                }
    14671466            }
    … …  
    16881687        print STDERR "desp_diff_singles: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n";
    16891688        }
    1690         return;
    1691     }
     1689    }
     1690    else {   
     1691        foreach my $object_row (@{ $object_ref }) {
     1692            my $this_object = shift @{ $object_row };
     1693            my $this_chunk = shift @{ $object_row };
     1694       
     1695            my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM ";
     1696            $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
     1697            $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1698            $input_sth .=   " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs ";
     1699
     1700            my $warps = $db->selectall_arrayref( $input_sth );
     1701
     1702            # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
     1703            my %comment_hash = ();
     1704            foreach my $this_warp (@{ $warps }) {
     1705                my $this_comment = ${ $this_warp }[3];
     1706                my $this_exp_id  = ${ $this_warp }[0];
     1707                my $this_quality = ${ $this_warp }[5];
     1708                if ($this_quality != 0) {
     1709                    print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
     1710                }
     1711                else {
     1712                    $comment_hash{$this_comment} = $this_exp_id;
     1713                }
     1714            }
     1715            # Exclude any warps that are not stored in the comment_hash.
     1716            my @keep_warps = ();
     1717            foreach my $this_warp (@{ $warps }) {
     1718                my $this_comment = ${ $this_warp }[3];
     1719                my $this_exp_id  = ${ $this_warp }[0];
     1720                if ((exists($comment_hash{$this_comment}))&&
     1721                    ($comment_hash{$this_comment} == $this_exp_id)) {
     1722                    push @keep_warps, $this_warp;
     1723                    $this_date  = ${ $this_warp }[2];
     1724                    $chunk_name  = ${ $this_warp }[6];
     1725                }
     1726                else {
     1727                    print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n";
     1728                }
     1729            }
     1730            @{ $warps } = @keep_warps;
     1731
     1732            #find the time of the most recent exposure in that chunk
     1733            my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' ";
     1734            my $chunk = $db->selectall_arrayref( $input_chunk );
     1735            foreach my $this_chunk (@{ $chunk }) {
     1736                $this_date = ${ $this_chunk }[0];
     1737            }
     1738 
     1739            #compute and store some stats for potential on-the-fly desperate diffs
     1740            my $nwarps = ($#{ $warps } + 1);
     1741            my $timediff = 0;
     1742            my $now=DateTime->now;
     1743            $now->set_time_zone("UTC");
     1744
     1745            if($this_date) {
     1746                if($this_date=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/) {
     1747                    my $dt=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
     1748                    my $difference=$now->delta_ms($dt);
     1749                    $timediff=($difference->hours) + (($difference->minutes) + ($difference->seconds)/60.)/60.;
     1750                }
     1751            }
     1752
     1753            #Consider the special case of on-the-fly desperate diffs
     1754            #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old
     1755            if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) {
     1756                print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n";
     1757                next;
     1758            }
     1759            if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) {
     1760                next;
     1761            }
     1762
     1763            # We are attempting to do the missing diffs, so reverse the list of retained warps.
     1764            # Good objects with all visits will be skipped due to the duplicate check.
     1765            # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs.
     1766            @{ $warps } = reverse @keep_warps;
     1767
     1768            # Exclude the last entry if we do not have an even number of warps.
     1769            if (($#{ $warps } + 1) % 2 != 0) {
     1770                print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!";
     1771                my $rejected_warp = pop @{ $warps };
     1772                my $rejected_exp_id = ${ $rejected_warp }[0];
     1773                print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
     1774            }
     1775       
     1776            while ($#{ $warps } > -1) {
     1777                # The array is sorted in pairs of input/template.
     1778                my $template_warp = shift @{ $warps };
     1779                my $input_warp = shift @{ $warps };
    16921780   
    1693     foreach my $object_row (@{ $object_ref }) {
    1694         my $this_object = shift @{ $object_row };
    1695         my $this_chunk = shift @{ $object_row };
     1781                my $input_exp_id = ${ $input_warp }[0];
     1782                my $input_comment = ${ $input_warp }[3];
     1783
     1784                my $template_exp_id = ${ $template_warp }[0];
     1785                my $template_comment = ${ $template_warp }[3];
     1786
     1787                my $input_warp_id = ${ $input_warp }[1];
     1788                my $template_warp_id = ${ $template_warp }[1];
     1789
     1790                my $input_warp_state = ${ $input_warp }[4];
     1791                my $template_warp_state = ${ $template_warp }[4];
     1792
     1793                my $input_warp_camQuality = ${ $input_warp }[5];
     1794                my $template_warp_camQuality = ${ $template_warp }[5];
     1795
     1796                $Npotential++;
     1797
     1798                unless (defined($input_warp_id) && defined($template_warp_id) &&
     1799                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
     1800                    print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
     1801                    if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
     1802                        # This should now never be reached.
     1803                        # CZW: Trigger backup plan here?  Or simply set up framework?
     1804                        print STDERR "  ...but this is due to a camera stage astrometry quality\n";
     1805                        $Npotential--;
     1806                    }
     1807                    next;
     1808                }
     1809
     1810                if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
     1811                    $Nqueued++;
     1812                    print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
     1813                    next;
     1814                }
     1815                else {
     1816                    print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
     1817                }
     1818           
     1819                my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
     1820                $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
     1821                $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
     1822                $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures
     1823                $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
     1824                $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
     1825                if (defined($reduction)) {
     1826                    $cmd .= " -set_reduction $reduction ";
     1827                }
    16961828       
    1697         my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM ";
    1698         $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
    1699         $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
    1700         $input_sth .=   " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs ";
    1701 
    1702         my $warps = $db->selectall_arrayref( $input_sth );
    1703 
    1704         # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
    1705         my %comment_hash = ();
    1706         foreach my $this_warp (@{ $warps }) {
    1707             my $this_comment = ${ $this_warp }[3];
    1708             my $this_exp_id  = ${ $this_warp }[0];
    1709             my $this_quality = ${ $this_warp }[5];
    1710             if ($this_quality != 0) {
    1711                 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
    1712             }
    1713             else {
    1714                 $comment_hash{$this_comment} = $this_exp_id;
    1715             }
    1716         }
    1717         # Exclude any warps that are not stored in the comment_hash.
    1718         my @keep_warps = ();
    1719         foreach my $this_warp (@{ $warps }) {
    1720             my $this_comment = ${ $this_warp }[3];
    1721             my $this_exp_id  = ${ $this_warp }[0];
    1722             if ((exists($comment_hash{$this_comment}))&&
    1723                 ($comment_hash{$this_comment} == $this_exp_id)) {
    1724                 push @keep_warps, $this_warp;
    1725                 $this_date  = ${ $this_warp }[2];
    1726                 $chunk_name  = ${ $this_warp }[6];
    1727             }
    1728             else {
    1729                 print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n";
    1730             }
    1731         }
    1732         @{ $warps } = @keep_warps;
    1733 
    1734         #find the time of the most recent exposure in that chunk
    1735         my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' ";
    1736         my $chunk = $db->selectall_arrayref( $input_chunk );
    1737         foreach my $this_chunk (@{ $chunk }) {
    1738             $this_date = ${ $this_chunk }[0];
     1829                if (defined($pretend)) {
     1830                    $cmd .= ' -pretend ';
     1831                }
     1832                if ($debug == 1) {
     1833                    $cmd .= ' -pretend ';
     1834                    print STDERR "desp_diff_singles: $cmd\n";
     1835                    print STDERR " $input_warp_id $template_warp_id\n";
     1836                }
     1837           
     1838                if (($debug == 0)&&(!defined($pretend))) {
     1839                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1840                        run ( command => $cmd, verbose => $verbose );
     1841                    unless ($success) {
     1842                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1843                        &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1844                    }
     1845                    $Nqueued++;
     1846                }
     1847            }
    17391848        }
    1740  
    1741         #compute and store some stats for potential on-the-fly desperate diffs
    1742         my $nwarps = ($#{ $warps } + 1);
    1743         my $timediff = 0;
    1744         my $now=DateTime->now;
    1745         $now->set_time_zone("UTC");
    1746 
    1747         if($this_date) {
    1748             if($this_date=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/) {
    1749                 my $dt=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
    1750                 my $difference=$now->delta_ms($dt);
    1751                 $timediff=($difference->hours) + (($difference->minutes) + ($difference->seconds)/60.)/60.;
    1752             }
    1753         }
    1754 
    1755         #Consider the special case of on-the-fly desperate diffs
    1756         #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old
    1757         if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) {
    1758             print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n";
    1759             next;
    1760         }
    1761         if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) {
    1762             next;
    1763         }
    1764 
    1765         # We are attempting to do the missing diffs, so reverse the list of retained warps.
    1766         # Good objects with all visits will be skipped due to the duplicate check.
    1767         # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs.
    1768         @{ $warps } = reverse @keep_warps;
    1769 
    1770         # Exclude the last entry if we do not have an even number of warps.
    1771         if (($#{ $warps } + 1) % 2 != 0) {
    1772             print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!";
    1773             my $rejected_warp = pop @{ $warps };
    1774             my $rejected_exp_id = ${ $rejected_warp }[0];
    1775             print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
    1776         }
    1777        
    1778         while ($#{ $warps } > -1) {
    1779             # The array is sorted in pairs of input/template.
    1780             my $template_warp = shift @{ $warps };
    1781             my $input_warp = shift @{ $warps };
    1782 
    1783             my $input_exp_id = ${ $input_warp }[0];
    1784             my $input_comment = ${ $input_warp }[3];
    1785 
    1786             my $template_exp_id = ${ $template_warp }[0];
    1787             my $template_comment = ${ $template_warp }[3];
    1788 
    1789             my $input_warp_id = ${ $input_warp }[1];
    1790             my $template_warp_id = ${ $template_warp }[1];
    1791 
    1792             my $input_warp_state = ${ $input_warp }[4];
    1793             my $template_warp_state = ${ $template_warp }[4];
    1794 
    1795             my $input_warp_camQuality = ${ $input_warp }[5];
    1796             my $template_warp_camQuality = ${ $template_warp }[5];
    1797 
    1798             $Npotential++;
    1799 
    1800             unless (defined($input_warp_id) && defined($template_warp_id) &&
    1801                     ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
    1802                 print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
    1803                 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
    1804                     # This should now never be reached.
    1805                     # CZW: Trigger backup plan here?  Or simply set up framework?
    1806                     print STDERR "  ...but this is due to a camera stage astrometry quality\n";
    1807                     $Npotential--;
    1808                 }
    1809                 next;
    1810             }
    1811 
    1812             if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
    1813                 $Nqueued++;
    1814                 print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
    1815                 next;
    1816             }
    1817             else {
    1818                 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
    1819             }
    1820            
    1821             my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
    1822             $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
    1823             $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
    1824             $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures
    1825             $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
    1826             $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
    1827             if (defined($reduction)) {
    1828                 $cmd .= " -set_reduction $reduction ";
    1829             }
    1830        
    1831             if (defined($pretend)) {
    1832                 $cmd .= ' -pretend ';
    1833             }
    1834             if ($debug == 1) {
    1835                 $cmd .= ' -pretend ';
    1836                 print STDERR "desp_diff_singles: $cmd\n";
    1837                 print STDERR " $input_warp_id $template_warp_id\n";
    1838             }
    1839            
    1840             if (($debug == 0)&&(!defined($pretend))) {
    1841                 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    1842                     run ( command => $cmd, verbose => $verbose );
    1843                 unless ($success) {
    1844                     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    1845                     &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
    1846                 }
    1847                 $Nqueued++;
    1848             }
    1849         }
    18501849    }
    18511850    $metadata_out{nsDiffPotential} += $Npotential;
    … …  
    19041903        print STDERR "desp_diff_queue: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n";
    19051904        }
    1906         return;
    1907     }
    1908    
    1909     foreach my $object_row (@{ $object_ref }) {
    1910         my $this_object = shift @{ $object_row };
    1911         my $this_chunk = shift @{ $object_row };
     1905    }
     1906    else {
     1907        foreach my $object_row (@{ $object_ref }) {
     1908            my $this_object = shift @{ $object_row };
     1909            my $this_chunk = shift @{ $object_row };
    19121910       
    1913         my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM ";
    1914         $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
    1915         $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
    1916         $input_sth .=   " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs ";
    1917 
    1918         my $warps = $db->selectall_arrayref( $input_sth );
    1919 
    1920         # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
    1921         my %comment_hash = ();
    1922         foreach my $this_warp (@{ $warps }) {
    1923             my $this_comment = ${ $this_warp }[3];
    1924             my $this_exp_id  = ${ $this_warp }[0];
    1925             my $this_quality = ${ $this_warp }[5];
    1926             if ($this_quality != 0) {
    1927                 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
    1928             }
    1929             else {
    1930                 $comment_hash{$this_comment} = $this_exp_id;
    1931             }
    1932         }
    1933         # Exclude any warps that are not stored in the comment_hash.
    1934         my @keep_warps = ();
    1935         foreach my $this_warp (@{ $warps }) {
    1936             my $this_comment = ${ $this_warp }[3];
    1937             my $this_exp_id  = ${ $this_warp }[0];
    1938             if ((exists($comment_hash{$this_comment}))&&
    1939                 ($comment_hash{$this_comment} == $this_exp_id)) {
    1940                 push @keep_warps, $this_warp;
    1941             }
    1942             else {
    1943                 print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n";
    1944             }
    1945         }
    1946         # We are attempting to do the missing diffs, so reverse the list of retained warps.
    1947         # Good objects with all visits will be skipped due to the duplicate check.
    1948         # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs.
    1949         @{ $warps } = reverse @keep_warps;
     1911            my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM ";
     1912            $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
     1913            $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1914            $input_sth .=   " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs ";
     1915
     1916            my $warps = $db->selectall_arrayref( $input_sth );
     1917
     1918            # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
     1919            my %comment_hash = ();
     1920            foreach my $this_warp (@{ $warps }) {
     1921                my $this_comment = ${ $this_warp }[3];
     1922                my $this_exp_id  = ${ $this_warp }[0];
     1923                my $this_quality = ${ $this_warp }[5];
     1924                if ($this_quality != 0) {
     1925                    print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
     1926                }
     1927                else {
     1928                    $comment_hash{$this_comment} = $this_exp_id;
     1929                }
     1930            }
     1931            # Exclude any warps that are not stored in the comment_hash.
     1932            my @keep_warps = ();
     1933            foreach my $this_warp (@{ $warps }) {
     1934                my $this_comment = ${ $this_warp }[3];
     1935                my $this_exp_id  = ${ $this_warp }[0];
     1936                if ((exists($comment_hash{$this_comment}))&&
     1937                    ($comment_hash{$this_comment} == $this_exp_id)) {
     1938                    push @keep_warps, $this_warp;
     1939                }
     1940                else {
     1941                    print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n";
     1942                }
     1943            }
     1944            # We are attempting to do the missing diffs, so reverse the list of retained warps.
     1945            # Good objects with all visits will be skipped due to the duplicate check.
     1946            # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs.
     1947            @{ $warps } = reverse @keep_warps;
    19501948                   
    1951         # Exclude the last entry if we do not have an even number of warps.
    1952         if (($#{ $warps } + 1) % 2 != 0) {
    1953             print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
    1954             if ($#{ $warps} + 1 == 1) {
    1955                 print STDERR ": I can do no diffs with only one exposure.\n";
    1956                 next;
    1957             }
    1958             else {
    1959                 my $rejected_warp = pop @{ $warps };
    1960                 my $rejected_exp_id = ${ $rejected_warp }[0];
    1961                 print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
    1962             }
    1963         }
     1949            # Exclude the last entry if we do not have an even number of warps.
     1950            if (($#{ $warps } + 1) % 2 != 0) {
     1951                print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
     1952                if ($#{ $warps} + 1 == 1) {
     1953                    print STDERR ": I can do no diffs with only one exposure.\n";
     1954                    next;
     1955                }
     1956                else {
     1957                    my $rejected_warp = pop @{ $warps };
     1958                    my $rejected_exp_id = ${ $rejected_warp }[0];
     1959                    print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
     1960                }
     1961            }
    19641962       
    1965         while ($#{ $warps } > -1) {
    1966             # In this mode, the array is sorted in pairs of template/input, counter to the standard queue.
    1967             my $template_warp = shift @{ $warps };
    1968             my $input_warp = shift @{ $warps };
    1969 
    1970             my $input_exp_id = ${ $input_warp }[0];
    1971             my $input_comment = ${ $input_warp }[3];
    1972 
    1973             my $template_exp_id = ${ $template_warp }[0];
    1974             my $template_comment = ${ $template_warp }[3];
    1975 
    1976             my $input_warp_id = ${ $input_warp }[1];
    1977             my $template_warp_id = ${ $template_warp }[1];
    1978 
    1979             my $input_warp_state = ${ $input_warp }[4];
    1980             my $template_warp_state = ${ $template_warp }[4];
    1981 
    1982             my $input_warp_camQuality = ${ $input_warp }[5];
    1983             my $template_warp_camQuality = ${ $template_warp }[5];
    1984 
    1985             $Npotential++;
    1986 
    1987             unless (defined($input_warp_id) && defined($template_warp_id) &&
    1988                     ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
    1989                 print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
    1990                 if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
    1991                     # This should now never be reached.
    1992                     # CZW: Trigger backup plan here?  Or simply set up framework?
    1993                     print STDERR "  ...but this is due to a camera stage astrometry quality\n";
    1994                     $Npotential--;
    1995                 }
    1996                 next;
    1997             }
    1998 
    1999             if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
    2000                 $Nqueued++;
    2001                 print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
    2002                 next;
    2003             }
    2004             else {
    2005                 print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
    2006             }
     1963            while ($#{ $warps } > -1) {
     1964                # In this mode, the array is sorted in pairs of template/input, counter to the standard queue.
     1965                my $template_warp = shift @{ $warps };
     1966                my $input_warp = shift @{ $warps };
     1967
     1968                my $input_exp_id = ${ $input_warp }[0];
     1969                my $input_comment = ${ $input_warp }[3];
     1970
     1971                my $template_exp_id = ${ $template_warp }[0];
     1972                my $template_comment = ${ $template_warp }[3];
     1973
     1974                my $input_warp_id = ${ $input_warp }[1];
     1975                my $template_warp_id = ${ $template_warp }[1];
     1976
     1977                my $input_warp_state = ${ $input_warp }[4];
     1978                my $template_warp_state = ${ $template_warp }[4];
     1979
     1980                my $input_warp_camQuality = ${ $input_warp }[5];
     1981                my $template_warp_camQuality = ${ $template_warp }[5];
     1982
     1983                $Npotential++;
     1984
     1985                unless (defined($input_warp_id) && defined($template_warp_id) &&
     1986                        ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
     1987                    print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
     1988                    if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
     1989                        # This should now never be reached.
     1990                        # CZW: Trigger backup plan here?  Or simply set up framework?
     1991                        print STDERR "  ...but this is due to a camera stage astrometry quality\n";
     1992                        $Npotential--;
     1993                    }
     1994                    next;
     1995                }
     1996
     1997                if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
     1998                    $Nqueued++;
     1999                    print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
     2000                    next;
     2001                }
     2002                else {
     2003                    print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
     2004                }
    20072005           
    2008             my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
    2009             $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
    2010             $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
    2011             $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures
    2012             $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
    2013             $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
    2014 #               $cmd .= " -pretend ";
    2015             if (defined($reduction)) {
    2016                 $cmd .= " -set_reduction $reduction ";
    2017             }
    2018 
    2019             if (defined($pretend)) {
    2020                 $cmd .= ' -pretend ';
    2021             }
    2022             if ($debug == 1) {
    2023                 $cmd .= ' -pretend ';
    2024                 print STDERR "desp_diff_queue: $cmd\n";
    2025                 print STDERR " $input_warp_id $template_warp_id\n";
    2026             }
     2006                my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
     2007                $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
     2008                $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
     2009                $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures
     2010                $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
     2011                $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
     2012#                   $cmd .= " -pretend ";
     2013                if (defined($reduction)) {
     2014                    $cmd .= " -set_reduction $reduction ";
     2015                }
     2016
     2017                if (defined($pretend)) {
     2018                    $cmd .= ' -pretend ';
     2019                }
     2020                if ($debug == 1) {
     2021                    $cmd .= ' -pretend ';
     2022                    print STDERR "desp_diff_queue: $cmd\n";
     2023                    print STDERR " $input_warp_id $template_warp_id\n";
     2024                }
    20272025           
    2028             if (($debug == 0)&&(!defined($pretend))) {
    2029                 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    2030                     run ( command => $cmd, verbose => $verbose );
    2031                 unless ($success) {
    2032                     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    2033                     &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
    2034                 }
    2035                 $Nqueued++;
    2036             }
    2037         }
     2026                if (($debug == 0)&&(!defined($pretend))) {
     2027                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     2028                        run ( command => $cmd, verbose => $verbose );
     2029                    unless ($success) {
     2030                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     2031                        &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     2032                    }
     2033                    $Nqueued++;
     2034                }
     2035            }
     2036        }
    20382037    }
    20392038    $metadata_out{nsDiffPotential} += $Npotential;
Note: See TracChangeset for help on using the changeset viewer.