IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Aug 5, 2019, 3:35:37 PM (7 years ago)
Author:
tdeboer
Message:

on the fly desperate diffs added to nightly_science script

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/nightly_science.pl

    r40403 r40855  
    4646
    4747# Grab options
    48 my ( $date, $datetime, $now, $camera, $dbname, $logfile, $verbose, $manual);
     48my ( $date, $datetime, $now, $camera, $dbname, $logfile, $verbose, $manual,$desdiffdt);
    4949my ( $help, $isburning, $force_stack_count, $force_diff_count, $force_registration, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
    5050my ( $registration_status, $burntool_status, $observing_status, $old_date);
    … …  
    8585    'queue_dqstats'        => \$queue_dqstats,
    8686    'clean_old'            => \$clean_old,
     87    'desdiffdt=s'             => \$desdiffdt, # TdB20190529delay time for making desperate iddfs on the fly
    8788    ) or pod2usage ( 2 );
    8889pod2usage( -msg =>
    … …  
    100101           --this_filter_only     Process only a single filter.
    101102           --this_mode_only       Process only a single clean mode.
    102            --burntool_status       Display Nexp Nimfile Nburntooled Nqueued for check_chips.
     103           --burntool_status      Display Nexp Nimfile Nburntooled Nqueued for check_chips.
     104           --desdiffdt <hrs>      Default 3hrs.
    103105        Modes:
    104106           --check_stacks         Confirm that stacks can be built.
    … …  
    248250        $datetime = DateTime->now(time_zone => 'UTC');
    249251        $datetime->set_year($year);
     252        $datetime->set_day($day);
    250253        $datetime->set_month($month);
    251         $datetime->set_day($day);
    252254    }
    253255}   
    … …  
    291293    $metadata_out{nsRegistrationState} = 'REGISTERED';
    292294}
     295
     296if (defined($desdiffdt) == 0) {
     297        $desdiffdt = 1.5;
     298}
     299
     300#Build in a safety for desdiffdt
     301if ((defined($desdiffdt) == 1) && ($desdiffdt <= 0.5)) {
     302        $desdiffdt = 0.5;
     303}
     304
    293305#
    294306# Mode selection
    … …  
    13571369    my $cam_ref  = $db->selectall_arrayref( $cam_sth );
    13581370    my $warp_ref = $db->selectall_arrayref( $warp_sth );
     1371
    13591372    return($#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2);
    13601373}
    … …  
    14021415                diff_queue($date,$target,$filter,$pretend);
    14031416
     1417                #Queue up desperate diffs if the conditions are right for them 
     1418                if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) {
     1419                    desperate_diff_singles($date,$target,$filter,$pretend);
     1420                }
     1421
    14041422                if (defined($pretend)) {
    14051423                    add_to_macro_list('check_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter);
    … …  
    14851503   
    14861504    foreach my $object_row (@{ $object_ref }) {
    1487         my $this_object = shift @{ $object_row };
    1488 #       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
    1489 #       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
    1490 #       $input_sth .= " ORDER BY dateobs ";
    1491        
     1505        my $this_object = shift @{ $object_row };       
    14921506        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM ";
    14931507        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
    … …  
    16201634}
    16211635
    1622 
    1623 sub desperate_diff_queue {
     1636sub desperate_diff_singles {
    16241637    my $date = shift;
    16251638    my $target = shift;
    … …  
    16271640    my $pretend = shift;
    16281641    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
     1642    my $this_date = undef;
     1643    my $chunk_name = undef;
     1644    my ($year,$month,$day,$hour,$min,$sec);
    16291645
    16301646    my $db = init_gpc_db();
    … …  
    16371653    my $Npotential = 0;
    16381654    my $Nqueued = 0;
     1655
     1656    #Before considering making diffs, we should check if there are no issues with summit download and/or registration.
     1657    #summitExp has limited information, so see if the number of exps in rawExp and summitExp match up before most recent dateobs of chunk
     1658    #be careful here, since dateobs is not quite completely the same in both tables. Therefore, allow numbers to be off by 1
     1659    my $timenow=DateTime->now;
     1660    $timenow->subtract(minutes => 15);
     1661    $timenow->set_time_zone("UTC");
     1662
     1663    my $dateinit = $timenow->ymd;
     1664    my $raw_sth = "select count(dateobs),MAX(dateobs) FROM rawExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' ";
     1665    my ($nrawexps,$rawdate) = $db->selectrow_array( $raw_sth );
     1666
     1667    my $summit_sth = "select count(dateobs),MAX(dateobs) FROM summitExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' ";
     1668    my ($nsummitexps,$summitdate) = $db->selectrow_array( $summit_sth );
     1669 
     1670    my $summittimediff = 9999;
     1671    if(($rawdate)&&($summitdate)) {
     1672        $rawdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
     1673        my $date1=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
     1674 
     1675        $summitdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
     1676        my $date2=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
     1677       
     1678        my $difference=$date2->delta_ms($date1);
     1679
     1680        $summittimediff=(($difference->hours)*60.) + ($difference->minutes) + (($difference->seconds)/60.);
     1681    }
     1682
     1683    #Run some basic checks
     1684    my $noff = abs($nsummitexps-$nrawexps);
     1685    if (($summittimediff > 10.)||($noff >10)) {
     1686        if ($debug == 1) {
     1687        print STDERR "desp_diff_singles: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n";
     1688        }
     1689        return;
     1690    }
     1691   
     1692    foreach my $object_row (@{ $object_ref }) {
     1693        my $this_object = shift @{ $object_row };
     1694       
     1695        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM ";
     1696        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
     1697        $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1698        $input_sth .=   " ORDER BY dateobs ";
     1699
     1700        my $warps = $db->selectall_arrayref( $input_sth );
     1701
     1702        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
     1703        my %comment_hash = ();
     1704        foreach my $this_warp (@{ $warps }) {
     1705            my $this_comment = ${ $this_warp }[3];
     1706            my $this_exp_id  = ${ $this_warp }[0];
     1707            my $this_quality = ${ $this_warp }[5];
     1708            if ($this_quality != 0) {
     1709                print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
     1710            }
     1711            else {
     1712                $comment_hash{$this_comment} = $this_exp_id;
     1713            }
     1714        }
     1715        # Exclude any warps that are not stored in the comment_hash.
     1716        my @keep_warps = ();
     1717        foreach my $this_warp (@{ $warps }) {
     1718            my $this_comment = ${ $this_warp }[3];
     1719            my $this_exp_id  = ${ $this_warp }[0];
     1720            if ((exists($comment_hash{$this_comment}))&&
     1721                ($comment_hash{$this_comment} == $this_exp_id)) {
     1722                push @keep_warps, $this_warp;
     1723                $this_date  = ${ $this_warp }[2];
     1724                $chunk_name  = ${ $this_warp }[6];
     1725            }
     1726            else {
     1727                print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n";
     1728            }
     1729        }
     1730        @{ $warps } = @keep_warps;
     1731
     1732        #find the time of the most recent exposure in that chunk
     1733        my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' ";
     1734        my $chunk = $db->selectall_arrayref( $input_chunk );
     1735        foreach my $this_chunk (@{ $chunk }) {
     1736            $this_date = ${ $this_chunk }[0];
     1737        }
     1738 
     1739        #compute and store some stats for potential on-the-fly desperate diffs
     1740        my $nwarps = ($#{ $warps } + 1);
     1741        my $timediff = 0;
     1742        my $now=DateTime->now;
     1743        $now->set_time_zone("UTC");
     1744
     1745        if($this_date) {
     1746            if($this_date=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/) {
     1747                my $dt=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
     1748                my $difference=$now->delta_ms($dt);
     1749                $timediff=($difference->hours) + (($difference->minutes) + ($difference->seconds)/60.)/60.;
     1750            }
     1751        }
     1752
     1753        #Consider the special case of on-the-fly desperate diffs
     1754        #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old
     1755        if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) {
     1756            print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n";
     1757            next;
     1758        }
     1759        if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) {
     1760            next;
     1761        }
     1762
     1763        # We are attempting to do the missing diffs, so reverse the list of retained warps.
     1764        # Good objects with all visits will be skipped due to the duplicate check.
     1765        # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs.
     1766        @{ $warps } = reverse @keep_warps;
     1767
     1768        # Exclude the last entry if we do not have an even number of warps.
     1769        if (($#{ $warps } + 1) % 2 != 0) {
     1770            print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!";
     1771            my $rejected_warp = pop @{ $warps };
     1772            my $rejected_exp_id = ${ $rejected_warp }[0];
     1773            print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
     1774        }
     1775       
     1776        while ($#{ $warps } > -1) {
     1777            # The array is sorted in pairs of input/template.
     1778            my $template_warp = shift @{ $warps };
     1779            my $input_warp = shift @{ $warps };
     1780
     1781            my $input_exp_id = ${ $input_warp }[0];
     1782            my $input_comment = ${ $input_warp }[3];
     1783
     1784            my $template_exp_id = ${ $template_warp }[0];
     1785            my $template_comment = ${ $template_warp }[3];
     1786
     1787            my $input_warp_id = ${ $input_warp }[1];
     1788            my $template_warp_id = ${ $template_warp }[1];
     1789
     1790            my $input_warp_state = ${ $input_warp }[4];
     1791            my $template_warp_state = ${ $template_warp }[4];
     1792
     1793            my $input_warp_camQuality = ${ $input_warp }[5];
     1794            my $template_warp_camQuality = ${ $template_warp }[5];
     1795
     1796            $Npotential++;
     1797
     1798            unless (defined($input_warp_id) && defined($template_warp_id) &&
     1799                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
     1800                print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
     1801                if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
     1802                    # This should now never be reached.
     1803                    # CZW: Trigger backup plan here?  Or simply set up framework?
     1804                    print STDERR "  ...but this is due to a camera stage astrometry quality\n";
     1805                    $Npotential--;
     1806                }
     1807                next;
     1808            }
     1809
     1810            if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
     1811                $Nqueued++;
     1812                print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
     1813                next;
     1814            }
     1815            else {
     1816                print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
     1817            }
     1818           
     1819            my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
     1820            $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
     1821            $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
     1822            $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures
     1823            $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
     1824            $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
     1825            if (defined($reduction)) {
     1826                $cmd .= " -set_reduction $reduction ";
     1827            }
     1828       
     1829            if (defined($pretend)) {
     1830                $cmd .= ' -pretend ';
     1831            }
     1832            if ($debug == 1) {
     1833                $cmd .= ' -pretend ';
     1834                print STDERR "desp_diff_singles: $cmd\n";
     1835                print STDERR " $input_warp_id $template_warp_id\n";
     1836            }
     1837           
     1838            if (($debug == 0)&&(!defined($pretend))) {
     1839                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1840                    run ( command => $cmd, verbose => $verbose );
     1841                unless ($success) {
     1842                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1843                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1844                }
     1845                $Nqueued++;
     1846            }
     1847        }
     1848    }
     1849    $metadata_out{nsDiffPotential} += $Npotential;
     1850    $metadata_out{nsDiffQueued}    += $Nqueued;
     1851}
     1852
     1853sub desperate_diff_queue {
     1854    my $date = shift;
     1855    my $target = shift;
     1856    my $filter = shift;
     1857    my $pretend = shift;
     1858    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
     1859    my ($year,$month,$day,$hour,$min,$sec);
     1860
     1861    my $db = init_gpc_db();
     1862
     1863    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
     1864    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
     1865
     1866    my $object_ref = $db->selectall_arrayref( $obj_sth );
     1867
     1868    my $Npotential = 0;
     1869    my $Nqueued = 0;
     1870
     1871    #Before considering making diffs, we should check if there are no issues with summit download and/or registration.
     1872    #summitExp has limited information, so see if the number of exps in rawExp and summitExp match up before most recent dateobs of chunk
     1873    #be careful here, since dateobs is not quite completely the same in both tables. Therefore, allow numbers to be off by 1
     1874    my $timenow=DateTime->now;
     1875    $timenow->subtract(minutes => 15);
     1876    $timenow->set_time_zone("UTC");
     1877
     1878    my $dateinit = $timenow->ymd;
     1879    my $raw_sth = "select count(dateobs),MAX(dateobs) FROM rawExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' ";
     1880    my ($nrawexps,$rawdate) = $db->selectrow_array( $raw_sth );
     1881
     1882    my $summit_sth = "select count(dateobs),MAX(dateobs) FROM summitExp WHERE exp_name LIKE 'o%' AND dateobs > '$dateinit' AND dateobs <= '$timenow' ";
     1883    my ($nsummitexps,$summitdate) = $db->selectrow_array( $summit_sth );
     1884 
     1885    my $summittimediff = 9999;
     1886    if(($rawdate)&&($summitdate)) {
     1887        $rawdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
     1888        my $date1=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
     1889 
     1890        $summitdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
     1891        my $date2=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC");
     1892       
     1893        my $difference=$date2->delta_ms($date1);
     1894
     1895        $summittimediff=(($difference->hours)*60.) + ($difference->minutes) + (($difference->seconds)/60.);
     1896    }
     1897
     1898    #Run some basic download/registration checks
     1899    my $noff = abs($nsummitexps-$nrawexps);
     1900    if (($summittimediff > 10.)||($noff >10)) {
     1901        if ($debug == 1) {
     1902        print STDERR "desp_diff_queue: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n";
     1903        }
     1904        return;
     1905    }
    16391906   
    16401907    foreach my $object_row (@{ $object_ref }) {
Note: See TracChangeset for help on using the changeset viewer.