IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 39800


Ignore:
Timestamp:
Nov 3, 2016, 2:04:31 PM (10 years ago)
Author:
watersc1
Message:

Untested update to nightly science that should construct all the diffs that MOPS wants automatically, with no need to queue replacements.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20160809/ippScripts/scripts/nightly_science.pl

    r38247 r39800  
    14351435    if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') {
    14361436        foreach my $target (sort (keys %science_config)) {
    1437             if ($science_config{$target}{DIFFABLE} == 1) {
     1437            # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now.
     1438            if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) {
    14381439                foreach my $filter (@filter_list) {
    1439                     multi_date_diff_queue($date,$target,$filter,$pretend);
    1440                 }
    1441             }
    1442             if ((defined($science_config{$target}{OFFNIGHT_DIFFS})) && ($science_config{$target}{OFFNIGHT_DIFFS} == 1)) {
    1443                 foreach my $filter (@filter_list) {
    1444                     offnight_diff_queue($date,$target,$filter,$pretend);
    1445                 }
    1446             }
     1440                    # This one needs to return a state to see if we need to wait on stacking before checking again.
     1441                    $metadata_out{nsDiffState} = desperate_diff_queue($date,$target,$filter,$pretend);
     1442                }
     1443            }
     1444
    14471445            if ((defined($science_config{$target}{SELF_WSDIFFS})) && ($science_config{$target}{SELF_WSDIFFS} == 1)) {
    14481446                foreach my $filter (@filter_list) {
    … …  
    14981496            my $this_comment = ${ $this_warp }[3];
    14991497            my $this_exp_id  = ${ $this_warp }[0];
    1500             $comment_hash{$this_comment} = $this_exp_id;
    1501         }
    1502        
     1498            my $this_quality = ${ $this_warp }[5];
     1499            if ($this_quality != 0) {
     1500                print STDERR "diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
     1501            }
     1502            else {
     1503                $comment_hash{$this_comment} = $this_exp_id;
     1504            }
     1505        }
     1506        # Exclude any warps that are not stored in the comment_hash.
     1507        my @keep_warps = ();
     1508        foreach my $this_warp (@{ $warps }) {
     1509            my $this_comment = ${ $this_warp }[3];
     1510            my $this_exp_id  = ${ $this_warp }[0];
     1511            if ($comment_hash{$this_comment} == $this_exp_id) {
     1512                push @keep_warps, $this_warp;
     1513            }
     1514            else {
     1515                print STDERR "diff_queue: excluding $this_exp_id for $this_object due to duplicate comment string $this_comment\n";
     1516            }
     1517        }
     1518        @{ $warps } = @keep_warps;
     1519
     1520        # Exclude the last entry if we do not have an even number of warps.
    15031521        if (($#{ $warps } + 1) % 2 != 0) {
    15041522            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
    … …  
    15081526            }
    15091527            else {
    1510                 print STDERR ": I should declare an exposure to be qualityy.\n";
    1511                 my @keep_warps = ();
    1512 #               print "@{ $warps }\n";
    1513                 foreach my $this_warp (@{ $warps }) {
    1514                     my $this_comment = ${ $this_warp }[3];
    1515                     my $this_exp_id  = ${ $this_warp }[0];
    1516                     if ($comment_hash{$this_comment} == $this_exp_id) {
    1517                         push @keep_warps, $this_warp;
    1518                     }
    1519                     else {
    1520                         print STDERR "diff_queue: excluding $this_exp_id for $this_object\n";
    1521                     }
    1522                 }
    1523                 @{ $warps } = @keep_warps;
    1524 #               print "@{ $warps }\n";
     1528                my $rejected_warp = pop @{ $warps };
     1529                my $rejected_exp_id = ${ $rejected_warp }[0];
     1530                print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
    15251531            }
    15261532        }
    15271533       
    15281534        while ($#{ $warps } > -1) {
     1535            # The array is sorted in pairs of input/template.
    15291536            my $input_warp = shift @{ $warps };
     1537            my $template_warp = shift @{ $warps };
     1538
    15301539            my $input_exp_id = ${ $input_warp }[0];
    15311540            my $input_comment = ${ $input_warp }[3];
    1532 
    1533            
    1534             my $template_warp = shift @{ $warps };
    15351541
    15361542            my $template_exp_id = ${ $template_warp }[0];
    … …  
    15511557                print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
    15521558                if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
     1559                    # This should now never be reached.
     1560                    # CZW: Trigger backup plan here?  Or simply set up framework?
    15531561                    print STDERR "  ...but this is due to a camera stage astrometry quality\n";
    15541562                    $Npotential--;
    … …  
    15611569                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
    15621570                next;
     1571            }
     1572            else {
     1573                print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
    15631574            }
    15641575           
    … …  
    16011612}
    16021613
     1614
     1615sub desperate_diff_queue {
     1616    my $date = shift;
     1617    my $target = shift;
     1618    my $filter = shift;
     1619    my $pretend = shift;
     1620    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
     1621
     1622    my $db = init_gpc_db();
     1623
     1624    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
     1625    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
     1626
     1627    my $object_ref = $db->selectall_arrayref( $obj_sth );
     1628
     1629    my $Npotential = 0;
     1630    my $Nqueued = 0;
     1631   
     1632    foreach my $object_row (@{ $object_ref }) {
     1633        my $this_object = shift @{ $object_row };
     1634       
     1635        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM ";
     1636        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
     1637        $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1638        $input_sth .=   " ORDER BY dateobs ";
     1639
     1640        my $warps = $db->selectall_arrayref( $input_sth );
     1641
     1642        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
     1643        my %comment_hash = ();
     1644        foreach my $this_warp (@{ $warps }) {
     1645            my $this_comment = ${ $this_warp }[3];
     1646            my $this_exp_id  = ${ $this_warp }[0];
     1647            my $this_quality = ${ $this_warp }[5];
     1648            if ($this_quality != 0) {
     1649                print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n";
     1650            }
     1651            else {
     1652                $comment_hash{$this_comment} = $this_exp_id;
     1653            }
     1654        }
     1655        # Exclude any warps that are not stored in the comment_hash.
     1656        my @keep_warps = ();
     1657        foreach my $this_warp (@{ $warps }) {
     1658            my $this_comment = ${ $this_warp }[3];
     1659            my $this_exp_id  = ${ $this_warp }[0];
     1660            if ($comment_hash{$this_comment} == $this_exp_id) {
     1661                push @keep_warps, $this_warp;
     1662            }
     1663            else {
     1664                print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to duplicate comment string $this_comment\n";
     1665            }
     1666        }
     1667        # We are attempting to do the missing diffs, so reverse the list of retained warps.
     1668        # Good objects with all visits will be skipped due to the duplicate check.
     1669        # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs.
     1670        @{ $warps } = reverse @keep_warps;
     1671                   
     1672        # Exclude the last entry if we do not have an even number of warps.
     1673        if (($#{ $warps } + 1) % 2 != 0) {
     1674            print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
     1675            if ($#{ $warps} + 1 == 1) {
     1676                print STDERR ": I can do no diffs with only one exposure.\n";
     1677                next;
     1678            }
     1679            else {
     1680                my $rejected_warp = pop @{ $warps };
     1681                my $rejected_exp_id = ${ $rejected_warp }[0];
     1682                print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n";
     1683            }
     1684        }
     1685       
     1686        while ($#{ $warps } > -1) {
     1687            # In this mode, the array is sorted in pairs of template/input, counter to the standard queue.
     1688            my $template_warp = shift @{ $warps };
     1689            my $input_warp = shift @{ $warps };
     1690
     1691            my $input_exp_id = ${ $input_warp }[0];
     1692            my $input_comment = ${ $input_warp }[3];
     1693
     1694            my $template_exp_id = ${ $template_warp }[0];
     1695           
     1696            my $input_warp_id = ${ $input_warp }[1];
     1697            my $template_warp_id = ${ $template_warp }[1];
     1698
     1699            my $input_warp_state = ${ $input_warp }[4];
     1700            my $template_warp_state = ${ $template_warp }[4];
     1701
     1702            my $input_warp_camQuality = ${ $input_warp }[5];
     1703            my $template_warp_camQuality = ${ $template_warp }[5];
     1704
     1705            $Npotential++;
     1706
     1707            unless (defined($input_warp_id) && defined($template_warp_id) &&
     1708                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
     1709                print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n";
     1710                if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) {
     1711                    # This should now never be reached.
     1712                    # CZW: Trigger backup plan here?  Or simply set up framework?
     1713                    print STDERR "  ...but this is due to a camera stage astrometry quality\n";
     1714                    $Npotential--;
     1715                }
     1716                next;
     1717            }
     1718
     1719            if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
     1720                $Nqueued++;
     1721                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
     1722                next;
     1723            }
     1724            else {
     1725                print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n";
     1726            }
     1727           
     1728            my $cmd = "$difftool -dbname $dbname  -definewarpwarp  ";
     1729            $cmd .= "-input_label $label  -template_label $label -good_frac 0.1 ";
     1730            $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
     1731            $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures
     1732            $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
     1733            $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
     1734#               $cmd .= " -pretend ";
     1735            if (defined($reduction)) {
     1736                $cmd .= " -set_reduction $reduction ";
     1737            }
     1738
     1739            if (defined($pretend)) {
     1740                $cmd .= ' -pretend ';
     1741            }
     1742            if ($debug == 1) {
     1743                $cmd .= ' -pretend ';
     1744                print STDERR "desp_diff_queue: $cmd\n";
     1745                print STDERR " $input_warp_id $template_warp_id\n";
     1746            }
     1747           
     1748            if (($debug == 0)&&(!defined($pretend))) {
     1749                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1750                    run ( command => $cmd, verbose => $verbose );
     1751                unless ($success) {
     1752                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1753                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1754                }
     1755                $Nqueued++;
     1756            }
     1757        }
     1758    }
     1759    $metadata_out{nsDiffPotential} += $Npotential;
     1760    $metadata_out{nsDiffQueued}    += $Nqueued;
     1761#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
     1762#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
     1763#      }       
     1764
     1765}
     1766
     1767
    16031768sub multi_date_diff_queue {
    16041769    my $date = shift;
    … …  
    16261791        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
    16271792        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
    1628         $input_sth .=   " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1793        $input_sth .=   " WHERE warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
    16291794        $input_sth .=   " ORDER BY dateobs ";
    16301795
Note: See TracChangeset for help on using the changeset viewer.