IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 20, 2010, 5:52:38 PM (16 years ago)
Author:
watersc1
Message:

updates to nightly science code to handle register failures and stacks and diffs

Location:
trunk/ippScripts/scripts
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/ipp_apply_burntool_single.pl

    r30049 r30120  
    178178        my $exp_id = shift;
    179179        my $class_id = shift;
    180         vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1);
     180        vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state 0 -set_state pending_burntool",1);
    181181    }
    182182    printf STDERR "$message\n";

  • trunk/ippScripts/scripts/nightly_science.pl

    r30067 r30120  
    396396    exit(0);
    397397}
    398 exit(10);
     398
    399399if (defined($check_confirm_stacks) || defined($test_mode)) {
    400400    $metadata_out{nsStackState} = 'CONFIRM_STACKING';
    … …  
    409409    exit(0);
    410410}
    411 
     411exit(10);
    412412if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) {
    413413    $metadata_out{nsSSState} = 'CHECKSWEETSPOT';
    … …  
    10971097    my $pretend = shift;
    10981098
     1099    my $Npotential = 0;
     1100    my $Nqueued = 0;
    10991101    foreach my $target (sort (keys %science_config)) {
    11001102        if ($science_config{$target}{STACKABLE} == 1) {
    … …  
    11201122                    next;
    11211123                }
     1124                $Npotential++;
    11221125                if ($Nalready != 0) {
     1126                    $Nqueued++;
    11231127                    if ($debug == 1) {
    11241128                        print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
    … …  
    11371141                    }
    11381142                    stack_queue($date,$target,$filter);
     1143                    $Nqueued ++;
    11391144                }
    11401145                if (defined($pretend)) {
    … …  
    11521157        }
    11531158    }
     1159    $metadata_out{nsStackPotential} = $Npotential;
     1160    $metadata_out{nsStackQueued}    = $Nqueued;
     1161    if (($Npotential == $Nqueued)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
     1162        $metadata_out{nsStackState} = 'FINISHED_STACKS';
     1163    }   
    11541164}
    11551165
    … …  
    12311241    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
    12321242
    1233     my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
     1243    my $count = 0;
     1244   
     1245    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
    12341246    my $data_ref = $db->selectall_arrayref( $sth );
    1235 
    1236     return($#{ $data_ref } + 1);
     1247    $count += $#{ $data_ref } + 1;
     1248
     1249    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
     1250    $data_ref = $db->selectall_arrayref( $sth );
     1251    $count += $#{ $data_ref } + 1;
     1252
     1253    return($count);
    12371254}
    12381255
    … …  
    12761293            foreach my $filter (@filter_list) {
    12771294                my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter);
    1278                 if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
    1279                     if ($debug == 1) {
    1280                         print STDERR "execute_diffs: Target $target in filter $filter on $date is not fully processed. ($NprocChips $NprocWarps)\n";
    1281                     }
    1282                     $metadata_out{nsDiffState} = 'FORCETOWARP';
    1283                     next;
    1284                 }
     1295#               if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
     1296#                   if ($debug == 1) {
     1297#                       print STDERR "execute_diffs: Target $target in filter $filter on $date is not fully processed. ($NprocChips $NprocWarps)\n";
     1298#                   }
     1299#                     $metadata_out{nsDiffState} = 'FORCETOWARP';
     1300#                     next;
     1301#                 }
    12851302                if ($NprocChips == 0) {
    12861303                    if ($debug == 1) {
    … …  
    13261343
    13271344    my $object_ref = $db->selectall_arrayref( $obj_sth );
     1345
     1346    my $Npotential = 0;
     1347    my $Nqueued = 0;
    13281348   
    13291349    foreach my $object_row (@{ $object_ref }) {
    13301350        my $this_object = shift @{ $object_row };
    1331         my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
     1351        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
    13321352        $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
    13331353        $input_sth .= " ORDER BY dateobs ";
    13341354       
    13351355        my $warps = $db->selectall_arrayref( $input_sth );
     1356
     1357        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
     1358        my %comment_hash = ();
     1359        foreach my $this_warp (@{ $warps }) {
     1360            my $this_comment = ${ $this_warp }[3];
     1361            my $this_exp_id  = ${ $this_warp }[0];
     1362            $comment_hash{$this_comment} = $this_exp_id;
     1363        }
    13361364       
    13371365        if (($#{ $warps } + 1) % 2 != 0) {
    1338             print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps }\n";
    1339             next;
     1366            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
     1367            if ($#{ $warps} + 1 == 1) {
     1368                print STDERR ": I can do no diffs with only one exposure.\n";
     1369                next;
     1370            }
     1371            else {
     1372                print STDERR ": I should declare an exposure to be faulty.\n";
     1373                my @keep_warps = ();
     1374#               print "@{ $warps }\n";
     1375                foreach my $this_warp (@{ $warps }) {
     1376                    my $this_comment = ${ $this_warp }[3];
     1377                    my $this_exp_id  = ${ $this_warp }[0];
     1378                    if ($comment_hash{$this_comment} == $this_exp_id) {
     1379                        push @keep_warps, $this_warp;
     1380                    }
     1381                    else {
     1382                        print STDERR "diff_queue: excluding $this_exp_id for $this_object\n";
     1383                    }
     1384                }
     1385                @{ $warps } = @keep_warps;
     1386#               print "@{ $warps }\n";
     1387            }
    13401388        }
    13411389       
    13421390        while ($#{ $warps } > -1) {
    13431391            my $input_warp = shift @{ $warps };
     1392            my $input_exp_id = ${ $input_warp }[0];
     1393            my $input_comment = ${ $input_warp }[3];
     1394
     1395           
    13441396            my $template_warp = shift @{ $warps };
    1345             my $input_exp_id = ${ $input_warp }[0];
     1397
    13461398            my $template_exp_id = ${ $template_warp }[0];
    13471399           
    13481400            my $input_warp_id = ${ $input_warp }[1];
    13491401            my $template_warp_id = ${ $template_warp }[1];
    1350            
     1402
     1403            $Npotential++;
    13511404            if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
     1405                $Nqueued++;
    13521406                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n";
    13531407                next;
    … …  
    13661420                $cmd .= ' -pretend ';
    13671421                print STDERR "diff_queue: $cmd\n";
     1422                print STDERR " $input_warp_id $template_warp_id\n";
    13681423            }
    13691424           
    … …  
    13781433        }
    13791434    }
     1435    $metadata_out{nsDiffPotential} += $Npotential;
     1436    $metadata_out{nsDiffQueued}    += $Nqueued;
     1437    if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
     1438        $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
     1439    }   
     1440
    13801441}
    13811442           

  • trunk/ippScripts/scripts/register_imfile.pl

    r30113 r30120  
    5454) or pod2usage( 2 );
    5555
    56 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    57 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
     56my $ipprc = PS::IPP::Config->new() or my_die_for_add( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
     57$ipprc->redirect_output($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
    5858
    5959pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    … …  
    101101    print "STDOUT:\n$out1";
    102102    print "STDERR:\n$err1";
    103     &my_die("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
     103    &my_die_for_add("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
    104104
    105105    print "[Running " . join(' ', @command2) . "]\n";
    … …  
    110110    print "STDOUT:\n$out2";
    111111    print "STDERR:\n$err2";
    112     &my_die("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
     112    &my_die_for_add("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
    113113    chomp $out2;
    114114    $cmdflags = $out2;
    … …  
    162162
    163163# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type
    164 if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    165 if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    166 if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    167 if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    168 if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     164if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die_for_add ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     165if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die_for_add ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     166if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die_for_add ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     167if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die_for_add ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     168if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die_for_add ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    169169
    170170my $command = "$regtool -addprocessedimfile";
    … …  
    177177$command .= " -hostname $host" if defined $host;
    178178$command .= " -dbname $dbname" if defined $dbname;
    179 $command .= " -data_state check_burntool";
     179if (abs($burntoolState) == $burntoolStateGood) {
     180    $command .= " -data_state full";
     181}
     182else {
     183    $command .= " -data_state check_burntool";
     184}
    180185$command .= " $cmdflags";
    181186
    … …  
    236241# We now have an imfile in the database, check if we can burntool it.  If not, continue on.
    237242
    238 {
     243if (abs($burntoolState) != $burntoolStateGood) {
    239244    my $mdcParser  = PS::IPP::Metadata::Config->new;
    240245
    … …  
    256261    unless ($success) {
    257262        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    258         &my_die("Unable to perform regtool: $error_code",
    259                 $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR);
     263        &my_die_for_update("Unable to perform regtool: $error_code",
     264                           $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR);
    260265    }
    261266
    … …  
    270275           
    271276            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
    272             &my_die("Unable to parse output from regtool",
    273                     $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR) unless
    274                         defined $list;
     277            &my_die_for_update("Unable to parse output from regtool",
     278                               $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR) unless
     279                                   defined $list;
    275280            push @{ $exposures }, @$list;
    276281            @single = ();
    … …  
    353358}
    354359
    355 sub my_die
     360sub my_die_for_add
    356361{
    357362    my $msg = shift; # Warning message on die
    … …  
    384389    exit $exit_code;
    385390}
     391sub my_die_for_update
     392{
     393    my $msg = shift; # Warning message on die
     394    my $exp_id = shift;
     395    my $exp_name = shift;
     396    my $class_id = shift;
     397    my $uri = shift;
     398    my $exit_code = shift;
     399
     400    # for failed imfiles, we insert UNKNOWN for inst, telescope, class_id
     401
     402    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     403
     404    carp($msg);
     405    if (defined $exp_id && defined $tmp_class_id and not $no_update) {
     406        my $command = "$regtool -updateprocessedimfile";
     407        $command .= " -exp_id $exp_id";
     408        $command .= " -class_id $class_id";
     409        $command .= " -fault $exit_code";
     410        $command .= " -hostname $host" if defined $host;
     411        $command .= " -dbname $dbname" if defined $dbname;
     412        print "Running: $command\n";
     413        system($command);
     414    }
     415    exit $exit_code;
     416}
    386417
    387418# Pau.
Note: See TracChangeset for help on using the changeset viewer.