IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jun 10, 2010, 6:28:51 PM (16 years ago)
Author:
watersc1
Message:

Skycell Summary and stack Association stuff should be finished. I'll merge and do final tests on monday.

Location:
branches/czw_branch/20100519
Files:
8 edited

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20100519

  • branches/czw_branch/20100519/pstamp/scripts/detectability_respond.pl

    r27859 r28304  
    226226my @psphot_Qfact= ();
    227227my @psphot_flux = ();
     228my @psphot_error = ();
    228229
    229230foreach my $k (keys %image_list_hash) {
    … …  
    231232        my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf";
    232233       
    233         my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE});
     234        my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE});
    234235       
    235236        push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
    … …  
    238239        push @psphot_Qfact,   @{ $tmp_Qfact };
    239240        push @psphot_flux,    @{ $tmp_flux };
     241        push @psphot_error,   @{ $tmp_flux_error };
    240242    }
    241243    else {
    … …  
    245247        push @psphot_Qfact,   (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
    246248        push @psphot_flux,    (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
     249        push @psphot_error,    (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
    247250    }   
    248251}
    … …  
    252255                    $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0],
    253256                    $query{HEADER}{obscode}[0],
    254                     \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux);
     257                    \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux, \@psphot_error);
    255258# print "Wrote response file $output\n";
    256259#
    … …  
    406409    my @tmp_Qfact = ();
    407410    my @tmp_flux = ();
     411    my @tmp_flux_err = ();
    408412
    409413    my ($detect_F_col,$detect_N_col,$detect_flux_col,$detect_mag_col) = (-1, -1, -1, -1);
    … …  
    428432            { name => 'PSF_NPIX', type => '1J', writetype => TLONG },
    429433            { name => 'PSF_INST_MAG', type => '1E', writetype => TDOUBLE },
     434            { name => 'PSF_INST_MAG_SIG', type => '1E', writetype => TDOUBLE },
    430435            ];
    431436    }
    … …  
    435440            { name => 'PSF_NPIX', type => '1J', writetype => TLONG },
    436441            { name => 'PSF_INST_FLUX', type => '1E', writetype => TDOUBLE },
     442            { name => 'PSF_INST_FLUX_SIG', type => '1E', writetype => TDOUBLE },
    437443            ];
    438444    }
    … …  
    450456    $inFits->get_num_rows($numRows, $status) and check_fitsio($status);
    451457
     458    my $correct_error = 0;
    452459    foreach my $col (@$column_defs) {
    453460        my ($col_num,$col_type,$col_data);
    … …  
    469476            @tmp_flux = map { $_ = 10**(-0.4 * $_) } @{ $col_data };
    470477        }
     478        elsif ($col->{name} eq 'PSF_INST_MAG_SIG') {
     479            @tmp_flux_err = map { $_ = $_ / (2.5 * log10(exp(1))) } @{ $col_data };
     480            $correct_error = 1;
     481        }
    471482        elsif ($col->{name} eq 'PSF_INST_FLUX') {
    472483            @tmp_flux = @{ $col_data };
    473484        }
     485        elsif ($col->{name} eq 'PSF_INST_FLUX_SIG') {
     486            @tmp_flux_err = @{ $col_data };
     487        }
     488    }
     489    if ($correct_error) {
     490        for (my $c = 0; $c <= $#tmp_flux_err; $c++) {
     491            $tmp_flux_err[$c] = $tmp_flux_err[$c] * $tmp_flux[$c];
     492        }
    474493    }
    475494    $inFits->close_file( $status ) and check_fitsio($status);   
    476     return(\@tmp_Npix, \@tmp_Qfact, \@tmp_flux);
     495    return(\@tmp_Npix, \@tmp_Qfact, \@tmp_flux, \@tmp_flux_err);
    477496}
    478497
    … …  
    490509    my $psphot_Qfact_ref = shift;
    491510    my $psphot_flux_ref = shift;
     511    my $psphot_error_ref = shift;
    492512    my $status = 0;
    493513
    … …  
    504524        # flux of the target source
    505525        { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
     526        # error in the flux of the target source
     527        { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
    506528        ];
    507529   
    … …  
    543565        push @{$colData{'DETECT_F'}},    ${ $psphot_Qfact_ref }[$i];
    544566        push @{$colData{'TARGET_FLUX'}}, ${ $psphot_flux_ref }[$i];
     567        push @{$colData{'TARGET_FLUX_SIG'}}, ${ $psphot_error_ref }[$i];
    545568        unless (defined(${ $colData{'TARGET_FLUX'}}[-1])) {
    546569            $colData{'TARGET_FLUX'}[-1] = 0.0;
     570        }
     571        unless (defined(${ $colData{'TARGET_FLUX_SIG'}}[-1])) {
     572            $colData{'TARGET_FLUX_SIG'}[-1] = 0.0;
    547573        }
    548574    }

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_checkdependent.pl

    r28164 r28304  
    299299        my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
    300300        my $data = runToolAndParse($command, $verbose);
    301         my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR)
    302             if !$data or scalar @$data == 0;
     301        if (!$data or scalar @$data == 0) {
     302            # This happens if the chipProcessedImfile disappears which happened when earlier
     303            # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before
     304            # deleing the row.
     305            # Fault the jobs so that the Request can finish ...
     306            faultJobs('stop', undef, undef, $PSTAMP_GONE);
     307            # ... and fault the dependent so that we have a record of the error
     308            my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id",
     309                $PSTAMP_GONE);
     310        }
     311
    303312
    304313        my $chips_ready = 1;

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_finish.pl

    r28164 r28304  
    7676if ($product eq "NULL") {
    7777    # nothing more to do
    78     stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
     78    my_die("product is NULL!", $req_id, $PS_EXIT_PROG_ERROR);
    7979}
    8080
    … …  
    9292
    9393        if (!mkdir $outdir) {
    94             print STDERR "cannot create output directory $outdir";
    95             stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
     94            my_die("cannot create output directory $outdir",$req_id, $PS_EXIT_UNKNOWN_ERROR);
    9695        }
    9796
    9897
    9998    } elsif (! -d $outdir ) {
    100         # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the
    101         # request
    102         print STDERR "output directory $outdir exists but is not a directory";
    103         stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
     99        my_die("output directory $outdir exists but is not a directory", $req_id, $PS_EXIT_UNKNOWN_ERROR);
    104100    }
    105101
    106102    if (! -e $req_file ) {
    107         print STDERR "request file $req_file is missing\n";
    108         stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
     103        my_die("request file $req_file is missing", $req_id, $PS_EXIT_CONFIG_ERROR);
    109104    }
    110105
    … …  
    115110        # Since a request got queued, the request file must have been readable at some
    116111        # point
    117         print STDERR "failed to read request_file $req_file" ;
    118         stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
     112        my_die("failed to read request file $req_file", $req_id, $PS_EXIT_CONFIG_ERROR);
    119113    }
    120114
    … …  
    151145        my $output = join "", @$stdout_buf;
    152146        if (!$output) {
    153             if ($verbose) {
    154                 print STDERR "Request $req_id produced no jobs.\n"
    155             }
    156147            # This should not happen. A fake job should have been entered
    157             stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
     148            my_die("Request $req_id produced no jobs", $req_id, $PS_EXIT_PROG_ERROR);
    158149        } else {
    159 if (0) {
    160             my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
    161 
    162             my $jobs = parse_md_list($metadata);
    163 }
    164150            my $jobs = parse_md_fast($mdcParser, $output);
    165151
    … …  
    181167            # this request had a duplicate request name yet the parser didn't give
    182168            # it an "ERROR style name.
    183             stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
     169            my_die("duplicate request not given a request name", $req_id, $PS_EXIT_PROG_ERROR);
    184170        }
    185171        my ($row, $req_info, $project) = get_request_info($rows, $rownum);
    … …  
    187173        my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
    188174        my $image_db = $proj_hash->{dbname};
    189         if (!$image_db) {
    190             carp("failed to find imagedb for project: $project");
    191             if (!$fault) {
    192                 stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
    193             }
     175        if (!$image_db and !$fault) {
     176            # if project isn't resolvable, the paser should have faulted this job
     177            my_die("failed to find imagedb for project $project", $req_id, $PS_EXIT_CONFIG_ERROR);
    194178        }
    195179
    … …  
    273257                run(command => $command, verbose => $verbose);
    274258            unless ($success) {
    275                 print STDERR "Unable to perform $command error code: $error_code\n";
    276259                $request_fault = $error_code >> 8;
     260                my_die("Unable to perform $command error code: $error_code", $req_id, $request_fault);
    277261            }
    278262        }
    … …  
    282266        my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS";
    283267        $command .= " --link --datapath $outdir --ps0 $req_id";
    284 # XXX: let dsreg and config handle resolving dbname and dbserver
    285 #        $command .= " --dbname $dbname" if $dbname;
    286268
    287269        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    288270            run(command => $command, verbose => $verbose);
    289271        unless ($success) {
    290             #die("Unable to perform $command error code: $error_code");
    291             print STDERR "Unable to perform $command error code: $error_code\n";
    292272            $request_fault = $error_code >> 8;
    293             # fall through to stop the request
     273            my_die("Unable to perform $command error code: $error_code\n", $req_id, $request_fault);
    294274        }
    295275    }
    296276    # set the request's state to stop
    297277    {
    298         my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $request_fault";
     278        my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop";
    299279        $command   .= " -dbname $dbname" if $dbname;
    300280        $command   .= " -dbserver $dbserver" if $dbserver;
    … …  
    308288}
    309289
    310 sub stop_request_and_exit {
     290sub my_die {
     291    my $msg = shift;
    311292    my $req_id = shift;
    312293    my $fault  = shift;
    313294
    314     my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $fault";
     295    carp($msg);
     296
     297    my $command = "$pstamptool -updatereq -req_id $req_id -set_fault $fault";
    315298    $command   .= " -dbname $dbname" if $dbname;
    316299    $command   .= " -dbserver $dbserver" if $dbserver;
    … …  
    445428                return undef;
    446429            }
    447 if (0) {
    448             my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
    449 
    450             my $exposures = parse_md_list($metadata);
    451 }
    452430            my $exposures = parse_md_fast($mdcParser, $output);
    453431            my $numExp = @$exposures;

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_insert_request.pl

    r27211 r28304  
    114114
    115115exit 0;
    116 
    117 # Temporary hack
    118 # webrequest number is stored in a file in the current directory
    119 #
    120 # get this number from the database
     116# Ask the database for the next web request number
    121117sub get_webreq_num
    122118{
    123     my $filename = "$workdir/webreq_num.txt";
    124     if (! open IN, "+< $filename" ) {
    125         my $initial_num = 1;
    126         open IN, "> $filename" or die "can't open $filename";
    127         print IN "$initial_num\n" or die "failed to initialize $filename";
    128         close IN;
    129         return $initial_num;
     119    my $command = "$pstamptool -getwebrequestnum";
     120    $command .= " -dbname $dbname" if $dbname;
     121    $command .= " -dbserver $dbserver" if $dbserver;
     122    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     123        run(command => $command, verbose => $verbose);
     124    unless ($success) {
     125        print STDERR @$stderr_buf;
     126        die("Unable to perform pstamptool -getwebrequestnum: $error_code");
     127    }
     128    my $webreq_num = ${$stdout_buf}[0];
     129    chomp $webreq_num;
     130
     131    if (!$webreq_num) {
     132        die("pstamptool -getwebreqnum returned no value");
    130133    }
    131134
    132     my $webreq_num = <IN>;
    133     chomp $webreq_num;
     135    # print STDERR "webreq_num $webreq_num\n";
    134136
    135     print STDERR "$webreq_num\n" if $verbose;
    136 
    137     my $next = $webreq_num + 1;
    138     truncate IN, 0;
    139     seek IN, 0, 0;
    140     print IN "$next\n";
    141 
    142     close IN;
    143137    return $webreq_num;
    144138}

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_parser_run.pl

    r27897 r28304  
    198198$parse_cmd .= " --verbose" if $verbose;
    199199
    200 my $newState = "run";
     200my $newState;
    201201my $fault;
    202202{
    … …  
    224224    }
    225225
    226     unless ($success) {
     226    if ($success) {
     227        $newState = 'run';
     228    } else {
    227229        $fault = $error_code >> 8;
    228 #        $newState = "stop";
    229230    }
    230231}
    … …  
    234235#
    235236{
    236     my $command = "$pstamptool -updatereq -req_id $req_id -set_state $newState";
     237    my $command = "$pstamptool -updatereq -req_id $req_id";
     238    $command   .= " -set_state $newState" if $newState;
    237239    $command   .= " -set_outdir $outdir";
    238240    $command   .= " -set_reqType $reqType" if $reqType;

  • branches/czw_branch/20100519/pstamp/scripts/pstamp_webrequest.pl

    r27355 r28304  
    2323
    2424my $host = hostname();
    25 my $verbose = 0;
     25my $verbose = 1;
    2626my $dbname;
    2727my $dbserver;
    … …  
    152152exit 0;
    153153
    154 # Temporary hack
    155 # webrequest number is stored in a file in the current directory
    156 #
     154# Ask the database for the next web request number
    157155sub get_webreq_num
    158156{
    159     my $filename = "webreq_num.txt";
    160     if (! open IN, "+< $filename" ) {
    161         my $initial_num = 1;
    162         open IN, "> $filename" or die "can't open $filename";
    163         print IN "$initial_num\n" or die "failed to initialize $filename";
    164         close IN;
    165         return $initial_num;
     157    my $command = "$pstamptool -getwebrequestnum";
     158    $command .= " -dbname $dbname" if $dbname;
     159    $command .= " -dbserver $dbserver" if $dbserver;
     160    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     161        run(command => $command, verbose => $verbose);
     162    unless ($success) {
     163        print STDERR @$stderr_buf;
     164        die("Unable to perform pstamptool -getwebrequestnum: $error_code");
     165    }
     166    my $webreq_num = ${$stdout_buf}[0];
     167    chomp $webreq_num;
     168
     169    if (!$webreq_num) {
     170        die("pstamptool -getwebreqnum returned no value");
    166171    }
    167172
    168     my $webreq_num = <IN>;
    169     chomp $webreq_num;
     173    # print STDERR "webreq_num $webreq_num\n";
    170174
    171     print STDERR "$webreq_num\n" if $verbose;
    172 
    173     my $next = $webreq_num + 1;
    174     truncate IN, 0;
    175     seek IN, 0, 0;
    176     print IN "$next\n";
    177 
    178     close IN;
    179175    return $webreq_num;
    180176}

  • branches/czw_branch/20100519/pstamp/scripts/pstampparse.pl

    r28164 r28304  
    361361    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
    362362    if (!$proj_hash) {
    363         print STDERR "project $project not found\n" ;
    364363        foreach $row (@$rowList) {
    365364            insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
    … …  
    829828            run(command => $command, verbose => $verbose);
    830829        unless ($success) {
    831             my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
    832         }
    833         my $output = join "", @$stdout_buf;
    834         chomp $output;
    835         $dep_id = $output;
    836         #
    837         # XXX: need to fault the request or something
    838         my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id;
     830            my $fault = $error_code >> 8;
     831            print STDERR "$command failed with fault $fault\n";
     832            if ($fault >= 10) {
     833                $$r_dep_id = 0;
     834                $$r_fault = $fault;
     835                $$r_jobState = 'stop';
     836                return;
     837            }
     838            # for now just die. Later pstamptool will return the whole dependent so we can
     839            # examine the fault code
     840            my_die("$command failed with unexpected fault value: $fault", $PS_EXIT_UNKNOWN_ERROR);
     841        } else {
     842            my $output = join "", @$stdout_buf;
     843            chomp $output;
     844            $dep_id = $output;
     845            #
     846            # XXX: need to fault the request or something
     847            my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id;
     848        }
    839849    } else {
    840850        print STDERR "skipping $command\n";
Note: See TracChangeset for help on using the changeset viewer.