IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Apr 23, 2010, 11:33:14 AM (16 years ago)
Author:
bills
Message:

Enable the postage stamp server to generate stamps from cleaned up data.
"Works" for chip and warp stages. diff and maybe stack to do.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

    r27589 r27751  
    2727use PS::IPP::Config qw( :standard );
    2828use Carp;
     29use DateTime;
     30use File::Temp qw(tempfile);
     31use File::Basename;
     32use POSIX;
     33
     34my $save_temps = 0;
    2935
    3036# caches of camProcessedExp objects.
    … …  
    3440my %camRunByCamIDCache;
    3541
    36 ### my @images = locate_images($image_db, $req_type, $img_type, $id, $component,
    37 ###            $mjd_min, $mjd_max, $filter);
     42# cache of last project looked up
     43my $last_project = "";
     44
     45my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool);
     46my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump);
    3847
    3948sub locate_images {
    4049    my $ipprc    = shift;   # required
    41     my $image_db = shift;   # required
     50    my $imagedb = shift;   # required
     51    my $rowList  = shift;   # required
    4252    my $req_type = shift;   # required
    4353    my $img_type = shift;   # required
    … …  
    4757    my $option_mask  = shift;
    4858    my $need_magic = shift;
    49     my $x        = shift;
    50     my $y        = shift;
    5159    my $mjd_min  = shift;
    5260    my $mjd_max  = shift;
    … …  
    5664
    5765    # we die in response to bad data in request files
    58     # The wrapper script is responsible for updating the database
    59     die "Unknown req_type: $req_type"
     66    # The caller is responsible for updating the database
     67    # pstampparse.pl error checks now so this shouldn't happen
     68    &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR)
    6069        if ($req_type ne "byid") and
    6170           ($req_type ne "byexp") and
    … …  
    8493
    8594    if ($req_type eq "bycoord") {
    86         my $results = lookup_bycoord($ipprc, $image_db, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
     95        my $num_rows = scalar @$rowList;
     96        die "Unexpected number of rows found in rowList: $num_rows" if $num_rows != 1;
     97        my $row = $rowList->[0];
     98        my $x = $row->{CENTER_X};
     99        my $y = $row->{CENTER_Y};
     100        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
    87101        return $results;
    88102    }
    … …  
    93107        # in one place
    94108        $req_type = "bydiff";
    95         my $results = lookup_diff($ipprc, $image_db, $id, $component, 1, $option_mask, $img_type, $verbose);
     109        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose);
    96110        return $results;
    97111    }
    98112
    99113    if ($req_type eq "bydiff") {
    100         my $results = lookup_diff($ipprc, $image_db, $id, $component, 0, $option_mask, $img_type, $verbose);
     114        # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or
     115        # the image from the diffRun
     116        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose);
    101117        if (!$results) {
    102118            return undef;
    … …  
    130146            # fall though and lookup by stack_id
    131147        } else {
    132             # shouldn't I check this elsewhere?
     148            # This is checked this elsewhere?
    133149            print STDERR "Error: $img_type is an unknown image type\n";
    134150            return undef;
    … …  
    145161    }
    146162
    147     my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic,
    148         $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
     163    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component,
     164        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
    149165
    150166    return $results;
    151167}
     168
     169# The subroutine lookup handles byexp, byid, and byskycell lookups including lookups that are
     170# triggered by a bydiff request
    152171
    153172sub lookup
    154173{
    155174    my $ipprc    = shift;
    156     my $image_db = shift;
     175    my $rowList    = shift;
     176    my $imagedb = shift;
    157177    my $req_type = shift;
    158     my $img_type = shift;
     178    my $stage = shift;
    159179    my $id       = shift;
    160180    my $tess_id  = shift;
    … …  
    172192    my $use_imfile_id = ($req_type eq "byid") && ($option_mask & $PSTAMP_USE_IMFILE_ID);
    173193
    174     my $missing_tools;
    175     my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
    176     my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
    177     my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
    178     my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
    179     my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
    180     if ($missing_tools) {
    181         warn("Can't find required tools.");
    182         exit ($PS_EXIT_CONFIG_ERROR);
    183     }
    184194    my $command;
    185195    my $id_opt;     # option for the lookup
    … …  
    203213    my $magic_arg = $need_magic ? " -destreaked" : "";
    204214
     215    # all rows have the same center type
     216    my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
     217
    205218    my $component_args;
    206     if ($img_type eq "raw") {
     219    my $choose_components = 0;
     220    if ($stage eq "raw") {
    207221        $class_id = $component;
    208         $command = "$regtool -processedimfile -dbname $image_db";
     222        $command = "$regtool -processedimfile -dbname $imagedb";
    209223        # XXX: for now restrict lookups to type object
    210224        # are stamps of detrend exposures interesting?
    … …  
    214228        $want_astrom = 1;
    215229        $set_class_id = 1;
    216     } elsif ($img_type eq "chip") {
     230    } elsif ($stage eq "chip") {
    217231        $class_id = $component;
    218         $command = "$chiptool -processedimfile -dbname $image_db";
    219         $component_args = " -class_id $class_id" if $class_id;
     232        # if the request is such that it will yield a single image per "run" or the
     233        # center is specified in pixel coordinates
     234        # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then
     235        # choose the chips containing the center pixel by calling selectComponets() below
     236        if ($class_id or $use_imfile_id or !$skycenter) {
     237            $command = "$chiptool -processedimfile -dbname $imagedb";
     238            $component_args = " -class_id $class_id" if $class_id;
     239        } else {
     240            $command = "$chiptool -listrun -pstamp_order -dbname $imagedb";
     241            $choose_components = 1;
     242        }
    220243        $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
     244        # XXX: perhaps we should stop resolving images to this level here and do it at job run time.
     245        # With the mdc file it has all of the information that it needs.
    221246        $image_name   = "PPIMAGE.CHIP";
    222247        $mask_name    = "PPIMAGE.CHIP.MASK";
    … …  
    228253        $want_astrom  = 1;
    229254        $set_class_id = 1;
    230     } elsif ($img_type eq "warp") {
     255    } elsif ($stage eq "warp") {
    231256        $skycell_id = $component;
    232         $command = "$warptool -warped -dbname $image_db";
    233         $component_args = " -skycell_id $skycell_id" if $skycell_id;
     257        if ($skycell_id or $use_imfile_id or !$skycenter) {
     258            $command = "$warptool -warped -dbname $imagedb";
     259            $component_args = " -skycell_id $skycell_id" if $skycell_id;
     260        } else {
     261            $command = "$warptool -listrun -pstamp_order -dbname $imagedb";
     262            $choose_components = 1;
     263        }
    234264        $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
    235265        $image_name   = "PSWARP.OUTPUT";
    … …  
    239269        $psf_name     = "PSPHOT.PSF.SKY.SAVE";
    240270        $base_name    = "path_base"; # name of the field for the warptool output
    241     } elsif ($img_type eq "diff") {
     271    } elsif ($stage eq "diff") {
    242272        $skycell_id = $component;
    243         $command = "$difftool -diffskyfile -dbname $image_db";
    244         $component_args = " -skycell_id $skycell_id" if $skycell_id;
     273        if ($skycell_id or $use_imfile_id or !$skycenter) {
     274            $command = "$difftool -diffskyfile -dbname $imagedb";
     275            $component_args = " -skycell_id $skycell_id" if $skycell_id;
     276        } else {
     277            $command = "$difftool -listrun -pstamp_order -dbname $imagedb";
     278            $choose_components = 1;
     279        }
    245280        $id_opt = $use_imfile_id ? "-diff_skyfile_id" : "-diff_id";
    246281        $image_name  = "PPSUB.OUTPUT";
    … …  
    250285        $psf_name     = "PSPHOT.PSF.SKY.SAVE";
    251286        $base_name   = "path_base";
    252     } elsif ($img_type eq "stack") {
     287    } elsif ($stage eq "stack") {
    253288        $skycell_id = $component;
    254         $command = "$stacktool -sumskyfile -dbname $image_db";
     289        $command = "$stacktool -sumskyfile -dbname $imagedb";
    255290        $id_opt = "-stack_id";
    256291        $component_args = " -skycell_id $skycell_id" if $skycell_id;
    … …  
    272307        $base_name   = "path_base";
    273308    } else {
    274         die "Unknown img_type supplied: $img_type";
    275     }
    276 
     309        die "Unknown IMG_TYPe supplied: $stage";
     310    }
     311
     312    my $filter_runs = 0;
    277313    if ($req_type eq "byid") {
    278314        $command .= " $id_opt $id";
    … …  
    288324        # the reason as 'not destreaked'
    289325        $magic_arg = "";
     326        # remove duplicate runs for the same exposure.
     327        $filter_runs = 1;
    290328    } elsif ($req_type eq "byskycell") {
    291329        die "tess_id and component are required for byskycell" if !$tess_id or ! $skycell_id;
    292330        $command .= " -tess_id $tess_id -skycell_id $skycell_id";
    293331    } else {
    294         die "Unknown req_type supplied: $req_type";
    295     }
    296 
    297     if ($img_type ne "stack") {
     332        # this should be caught by caller
     333        &my_die("Unexpected req_type supplied: $req_type", $PS_EXIT_PROG_ERROR);
     334    }
     335
     336    if ($stage ne "stack") {
    298337        $command .= $magic_arg;
    299338        $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
    … …  
    304343    $command .= " -data_group $data_group" if !isnull($data_group);
    305344
    306     # run the tool and parse the output
    307     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    308                 run(command => $command, verbose => $verbose);
    309     unless ($success) {
    310         # not sure if we should die here
    311         print STDERR @$stderr_buf;
     345    my $images = runToolAndParse($command, $verbose);
     346
     347    if (!$images or scalar @$images == 0) {
    312348        return undef;
    313349    }
    314350
    315     my $buf = join "", @$stdout_buf;
    316     if (!$buf) {
    317         return;
    318     }
    319 
    320     my $mdcParser = PS::IPP::Metadata::Config->new;
    321     my $images = parse_md_fast($mdcParser, $buf)
    322         or die ("Unable to parse metadata config doc");
    323 
     351    if ($filter_runs) {
     352        # The image selectors are such that multiple runs my have be returned for the same exposure.
     353        # Return only the latest one.
     354        $images = filterRuns($stage, $need_magic, $images, $verbose);
     355    }
     356    if ($choose_components) {
     357        # the list of "images" is actually a list of "Runs"
     358        # match the coords in the rows to the components in the runs
     359        # returns an actual list of images
     360        $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
     361    } else {
     362        # put each of the rows into all of the the images
     363        setRowRefs($rowList, $images);
     364    }
     365
     366    my $camera;
    324367    my $output = [];
    325 
    326     my $camera;
    327368    foreach my $image (@$images) {
    328369        my $base;
    329370
    330371        next if $image->{fault};
    331         next if ($img_type ne "raw") and $image->{quality};
     372        next if ($stage ne "raw") and $image->{quality};
    332373
    333374        if ($base_name) {
    … …  
    361402        }
    362403        my $stage_id;
    363         if ($img_type eq "raw") {
     404        if ($stage eq "raw") {
    364405            $stage_id = $image->{exp_id};
    365         } elsif ($img_type eq "chip") {
     406        } elsif ($stage eq "chip") {
    366407            $stage_id = $image->{chip_id};
    367         } elsif ($img_type eq "warp") {
     408        } elsif ($stage eq "warp") {
    368409            $stage_id = $image->{warp_id};
    369         } elsif ($img_type eq "diff") {
     410        } elsif ($stage eq "diff") {
    370411            $stage_id = $image->{diff_id};
    371412            if ($inverse && $image->{bothways}) {
    … …  
    375416                $cmf_name    = "PPSUB.INVERSE.SOURCES";
    376417            }
    377         } elsif ($img_type eq "stack") {
     418        } elsif ($stage eq "stack") {
    378419            $stage_id = $image->{stack_id};
    379420        }
    380421        $out->{stage_id} = $stage_id;
    381         $out->{stage}    = $img_type;
    382         $out->{image_db} = $image_db;
     422        $out->{stage}    = $stage;
     423        $out->{imagedb} = $imagedb;
    383424
    384425        if ($base) {
    … …  
    390431            $out->{backmdl}= $ipprc->filename($backmdl_name, $base, $class_id) if $backmdl_name;
    391432        }
    392         $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom;
     433        if ($want_astrom and ! defined $out->{astrom}) {
     434            $out->{astrom} = find_astrometry($ipprc, $imagedb, $image, $verbose);
     435        }
    393436
    394437        push @$output, $out;
    … …  
    397440    return $output;
    398441}
     442
    399443sub lookup_diff {
    400444    my $ipprc    = shift;
    401     my $image_db = shift;
     445    my $rowList  = shift;
     446    my $imagedb = shift;
    402447    my $id       = shift;
    403448    my $skycell_id = shift;
    … …  
    409454    my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE;
    410455
    411     my $missing_tools;
    412     my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
    413     if ($missing_tools) {
    414         warn("Can't find required tools.");
    415         exit ($PS_EXIT_CONFIG_ERROR);
    416     }
    417 
    418     my $command = "$difftool -diffskyfile -dbname $image_db";
     456    my $command = "$difftool -diffskyfile -dbname $imagedb";
    419457   
    420458    if ($byid) {
    … …  
    425463    $command .= " -skycell_id $skycell_id" if $skycell_id;
    426464
    427     # run the tool and parse the output
    428     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    429                 run(command => $command, verbose => $verbose);
    430     unless ($success) {
    431         # not sure if we should die here
    432         print STDERR @$stderr_buf;
    433         return undef;
    434     }
    435 
    436     my $buf = join "", @$stdout_buf;
    437     if (!$buf) {
    438         return undef;
    439     }
    440 
    441     my $mdcParser = PS::IPP::Metadata::Config->new;
    442     my $images = parse_md_fast($mdcParser, $buf)
    443         or die ("Unable to parse metadata config doc");
     465    my $images = runToolAndParse($command, $verbose);
    444466
    445467    my $n = @$images;
    … …  
    527549
    528550        if ($img_type eq "diff") {
     551            my @imageList = ($image);
     552
     553            setRowRefs($rowList, \@imageList);
    529554            # the $image is going to be returned directly in this case so we need to duplicate
    530555            # some of processing that lookup does for other img_types
    … …  
    539564                $image->{stage_id} = $image->{diff_id};
    540565                $image->{stage}    = "diff";
    541                 $image->{image_db} = $image_db;
     566                $image->{imagedb} = $imagedb;
    542567                $image->{component} = $image->{skycell_id};
    543568            } else {
    … …  
    554579sub lookup_bycoord {
    555580    my $ipprc      = shift;
    556     my $image_db   = shift;
     581    my $rowList    = shift;
     582    my $imagedb    = shift;
    557583    my $img_type   = shift;
    558584    my $tess_id    = shift;
    … …  
    568594    my $verbose    = shift;
    569595
    570     my $missing_tools;
    571     my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
    572     my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
    573     my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
    574     my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
    575     if ($missing_tools) {
    576         warn("Can't find required tools.");
    577         exit ($PS_EXIT_CONFIG_ERROR);
    578     }
    579 
    580596    my $results = ();
    581597    if (($img_type eq "raw") or ($img_type eq "chip")) {
    582598
    583         my $runs = lookup_runs_by_camid_and_coords($ipprc, $image_db, $img_type,
     599        my $runs = lookup_runs_by_cam_id_and_coords($ipprc, $imagedb, $img_type,
    584600            $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose);
    585601
     602        # lookup is going to filter these we don't need to do it here
     603#        $runs = filterRuns($img_type, $need_magic, $runs, $verbose);
    586604        foreach my $run (@$runs) {
    587605            next if $component and ($run->{component} ne $component);
    588             my $these_results = lookup($ipprc, $image_db, "byid", $img_type, $run->{id},
     606            my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $run->{id},
    589607                $tess_id, $run->{component}, $need_magic,
    590608                $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
    … …  
    602620
    603621        foreach my $skycell (@$skycells) {
    604             my $these_results = lookup($ipprc, $image_db, "byskycell", $img_type, undef,
     622            my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef,
    605623                $skycell->{tess_id}, $skycell->{component}, $need_magic,
    606624                $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
    … …  
    614632}
    615633
    616 # lookup_runs_by_camid_and_coords
     634# lookup_runs_by_cam_id_and_coords
    617635# given an ra, dec, and optionally other paramters, find camera runs for exposures
    618636# that are within some distance of the coordinates
    619 sub lookup_runs_by_camid_and_coords {
     637sub lookup_runs_by_cam_id_and_coords {
    620638    my $ipprc      = shift;
    621     my $image_db   = shift;
     639    my $imagedb   = shift;
    622640    my $img_type   = shift;
    623641    my $ra         = shift;
    … …  
    631649
    632650    my $camruns;
    633     my $missing_tools;
    634     my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
    635     my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
    636     if ($missing_tools) {
    637         warn("Can't find required tools.");
    638         exit ($PS_EXIT_CONFIG_ERROR);
    639     }
    640651
    641652    {
    642         my $command = "$camtool -dbname $image_db -processedexp";
     653        my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
    643654           $command .= " -ra $ra -decl $dec -radius 1.6";
    644655           $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
    … …  
    652663           $command .= " -data_group $data_group" if $data_group;
    653664           $command .= " -destreaked" if $need_magic;
     665
    654666        # run the tool and parse the output
    655         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    656                     run(command => $command, verbose => $verbose);
    657         unless ($success) {
    658             print STDERR @$stderr_buf;
    659             return undef;
    660         }
    661         my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    662 
    663         my $output = join "", @$stdout_buf;
    664         if (!$output) {
    665             print STDERR "no output returned from $command\n" if $verbose;
    666             return undef;
    667         }
    668         $camruns = parse_md_fast($mdcParser, $output);
     667        $camruns = runToolAndParse($command, $verbose);
    669668    }
    670669    if (!$camruns) {
    … …  
    672671    }
    673672    my $runs;
     673    my $last_exp_id = 0;
    674674    foreach my $camRun (@$camruns) {
    675675        my $cam_id = $camRun->{cam_id};
    676 
    677         updateCamRunCache($camRun);
     676        my $exp_id = $camRun->{exp_id};
     677
     678        next if $exp_id eq $last_exp_id;
    678679
    679680        next if $camRun->{quality};
    680681        next if $camRun->{fault};
     682
     683        updateCamRunCache($camRun);
     684
     685        $last_exp_id = $exp_id;
     686
    681687        # XXX Use file rule
    682688        my $astrom = $camRun->{path_base} . ".smf";
    … …  
    684690        next if !$astrom_resolved;
    685691
     692        my $start_dvo = DateTime->now->mjd;
    686693        my $command = "$dvoImagesAtCoords -astrom $astrom_resolved $ra $dec";
    687694        # run the tool and parse the output
     695        my $dvo_verbose = 0;
    688696        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    689                     run(command => $command, verbose => $verbose);
     697                    run(command => $command, verbose => $dvo_verbose);
     698        my $dtime_dvo = (DateTime->now->mjd - $start_dvo)* 86400.;
     699#        print "Time to run dvoImagesAtCoords: $dtime_dvo\n";
    690700        unless ($success) {
    691701            my $result_code = $error_code >> 8;
    692702            if ($result_code == $PSTAMP_NO_OVERLAP) {
    693                 print STDERR "no overlap for $astrom\n" if $verbose;
     703                print "no overlap for $astrom\n" if $verbose;
    694704                next;
    695705            }
    … …  
    721731            die "unexpected output from dvoImagesAtCoords: $lines[0]";
    722732        }
     733
     734        print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n";
     735
    723736        # build the hash to return
    724737        my $run = {
    … …  
    732745        };
    733746        if ($img_type eq "chip") {
    734             $run->{id} = $run->{chip_id};
     747            $run->{id} = $camRun->{chip_id};
     748            $run->{state} = $camRun->{chip_state};
     749            $run->{magicked} = $camRun->{chip_magicked};
    735750        } else {
    736             $run->{id} = $run->{exp_id};
     751            $run->{id} = $camRun->{exp_id};
     752            $run->{state} = 'full';
     753            $run->{magicked} = $camRun->{raw_magicked};
    737754        }
    738755        push @$runs, $run;
    … …  
    749766    my $dec        = shift;
    750767    my $verbose    = shift;
    751 
    752     my $missing_tools;
    753     my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
    754     if ($missing_tools) {
    755         warn("Can't find required tools.");
    756         exit ($PS_EXIT_CONFIG_ERROR);
    757     }
    758768
    759769    $requested_tess_id = "" if isnull($requested_tess_id);
    … …  
    817827sub find_astrometry {
    818828    my $ipprc = shift;
    819     my $image_db = shift;
     829    my $imagedb = shift;
    820830    my $image = shift;      # hashref to output of the lookup tool
    821831    my $verbose = shift;
    … …  
    825835    my $exp_id = $image->{exp_id};
    826836    if (($exp_id eq $last_exp_id) and $lastAstromFile) {
    827         # running camtool 60 times is really expensive when the answer is the same
     837        # running camtool 60 times is really expensive when the answer is the same every time
    828838        return $lastAstromFile;
    829839    }
    … …  
    831841    $lastAstromFile = undef;
    832842
    833     my $missing_tools;
    834     my $camtool = can_run("camtool") or (warn "Can't find camtool" and $missing_tools = 1);
    835     my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
    836     if ($missing_tools) {
    837         warn("Can't find required tools.");
    838         exit ($PS_EXIT_CONFIG_ERROR);
    839     }
    840 
    841843    my $camRun = getCamRunByExpID($exp_id);
    842844    if (!$camRun) {
    843         my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id";
     845        my $command = "$camtool -dbname $imagedb -processedexp -exp_id $exp_id -pstamp_order";
    844846        # run the tool and parse the output
    845847        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    864866        # If there are multiple cam runs for this exposure, take the last completed one with good quality
    865867        # on the assumption that it has the best astrometry.
    866         while ($camRun = pop @$camruns) {
    867             last if (($camRun->{quality} eq 0) and ($camRun->{fault} eq 0));
     868        foreach $camRun (@$camruns) {
     869            last if (($camRun->{state} eq 'full') and ($camRun->{quality} eq 0) and ($camRun->{fault} eq 0));
    868870        }
    869871        # XXX: this looks like a bug at least if ASTROM.SOURCE eq PSASTRO.OUTPUT
    … …  
    959961# resolve_project()
    960962# get project specific information
    961 my $last_project = "";
    962963sub resolve_project {
    963964    my $ipprc = shift;
    964965    my $project_name = shift;
    965966
     967    findTools();
     968
    966969    if (!$project_name) {
    967970        carp ("project is not defined");
    … …  
    978981    my $verbose = 0;
    979982
    980     my $missing_tools;
    981     my $pstamptool = can_run("pstamptool") or (warn "Can't find pstamptool" and $missing_tools = 1);
    982     if ($missing_tools) {
    983         warn("Can't find required tools.");
    984         exit ($PS_EXIT_CONFIG_ERROR);
    985     }
    986 
    987983    my $command = "$pstamptool -project -name $project_name";
    988984    $command .= " -dbname $dbname" if defined $dbname;
    989985    $command .= " -dbserver $dbserver" if defined $dbserver;
    990     # run the tool and parse the output
    991     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    992                 run(command => $command, verbose => $verbose);
    993     unless ($success) {
    994         print STDERR @$stderr_buf;
     986
     987    my $proj_hash = runToolAndParse($command, $verbose);
     988    if (!$proj_hash or scalar @$proj_hash == 0) {
     989        print STDERR "Project $project_name not found\n";
    995990        return undef;
    996991    }
    997 
    998     my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    999 
    1000     my $output = join "", @$stdout_buf;
    1001     if (!$output) {
    1002         print STDERR "no output returned from $command\n" if $verbose;
    1003         return undef;
    1004     }
    1005     my $proj_hash = parse_md_fast($mdcParser, $output);
    1006992
    1007993    $last_project = $proj_hash->[0];
    … …  
    10351021sub getCamRunByExpID {
    10361022    my $exp_id = shift;
    1037     die "getCamRun: exp_id is nil" if !$exp_id;
     1023    &my_die ("getCamRun: exp_id is nil", $PS_EXIT_PROG_ERROR) if !$exp_id;
    10381024
    10391025    return $camRunByExpIDCache{$exp_id};
    10401026}
     1027
    10411028sub getCamRunByCamID {
    10421029    my $cam_id = shift;
    1043     die "getCamRun: cam_id is nil" if !$cam_id;
     1030    &my_die ("getCamRun: cam_id is nil", $PS_EXIT_PROG_ERROR) if !$cam_id;
    10441031
    10451032    return $camRunByCamIDCache{$cam_id};
    10461033}
     1034
     1035sub selectComponents {
     1036    my $ipprc = shift;
     1037    my $imagedb = shift;
     1038    my $req_type = shift;
     1039    my $stage  = shift;
     1040    my $rowList = shift;
     1041    my $runList = shift;
     1042    my $verbose = shift;
     1043    my $results = [];
     1044   
     1045    my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
     1046    my $npoints = 0;
     1047    foreach my $row (@$rowList) {
     1048        print $pointsList "$npoints $row->{CENTER_X} $row->{CENTER_Y}\n";
     1049        $npoints++;
     1050    }
     1051    close $pointsList;
     1052
     1053    # XXX: need to loop over these
     1054    # my $run = $runList->[0];
     1055    if (($req_type eq "byid") or ($req_type eq "byexp")) {
     1056        my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", "");
     1057        foreach my $run (@$runList) {
     1058            my $command = "$dvoImagesAtCoords -coords $pointsListName";
     1059            if (($stage eq "chip") or ($stage eq "raw")) {
     1060                # XXX: use file rule and handle cameras where the astrometry is solved at
     1061                # the chip stage.
     1062                $astrom_file = $run->{cam_path_base} . ".smf";
     1063                my $astrom_file_resolved = $ipprc->file_resolve($astrom_file);
     1064                $command .= " -astrom $astrom_file_resolved";
     1065            } else {
     1066                my $tess_id = $run->{tess_id};
     1067                if ($tess_id ne $last_tess_id) {
     1068                    $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id );
     1069                    $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs );
     1070                    $last_tess_id = $tess_id;
     1071                }
     1072                $command .= " -D CATDIR $tess_dir_abs"
     1073            }
     1074            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1075                 run(command => $command, verbose => $verbose);
     1076            unless ($success) {
     1077                # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP
     1078                #                     # That just means that the coordinate didn't match any image/skycell
     1079                if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) {
     1080                    print STDERR @$stderr_buf;
     1081                    my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR;
     1082                    &my_die( "dvoImagesAtCoords failed: $rc", $rc);
     1083                }
     1084            }
     1085            my %components;
     1086            my @lines = split "\n", join "", @$stdout_buf;
     1087            foreach my $line (@lines) {
     1088                my ($ptnum, undef, undef, $component) = split " ", $line;
     1089                my $ref = $components{$component};
     1090                if (!$ref) {
     1091                    $ref = $components{$component} = [];
     1092                }
     1093                push @$ref, $ptnum;
     1094            }
     1095            if ($verbose) {
     1096                foreach my $c (keys %components) {
     1097                    my $ref = $components{$c};
     1098                    print "component $c contains: ";
     1099                    foreach my $i (@$ref) {
     1100                        print "$i ";
     1101                    }
     1102                    print "\n";
     1103                }
     1104            }
     1105            # now find the images for this run
     1106            foreach my $c (keys %components) {
     1107                my $command;
     1108                my $stage_id;
     1109                if ($stage eq 'raw') {
     1110                    $stage_id = $run->{exp_id};
     1111                    $command = "$regtool -processedimfile -exp_id $stage_id -class_id $c";
     1112                } elsif ($stage eq 'chip') {
     1113                    $stage_id = $run->{chip_id};
     1114                    $command = "$chiptool -processedimfile -chip_id $stage_id -class_id $c";
     1115                } elsif ($stage eq 'warp') {
     1116                    $stage_id = $run->{warp_id};
     1117                    $command = "$warptool -warped -warp_id $stage_id -skycell_id $c";
     1118                } elsif ($stage eq 'diff') {
     1119                    $stage_id = $run->{diff_id};
     1120                    $command = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $c";
     1121                }
     1122                $command .= " -dbname $imagedb";
     1123                my $images = runToolAndParse($command, $verbose);
     1124                if (!defined $images) {
     1125                    # XXX: need to get the set error_code to $PSTAMP_NO_OVERLAP
     1126                    print "No components containing coordinates found for ${stage}_id $stage_id\n";
     1127                    foreach my $row (@$rowList) {
     1128                        $row->{error_code} = $PSTAMP_NO_OVERLAP;
     1129                    }
     1130                    next;
     1131                }
     1132                if (scalar @$images != 1) {
     1133                    my $num_images = scalar @$images;
     1134                    &my_die ("unexpected number of images returned: $num_images\n", $PS_EXIT_PROG_ERROR);
     1135                }
     1136                my $image = $images->[0];
     1137                   
     1138                my $ref = $components{$c};
     1139                $image->{row_index} = $ref;
     1140                if (($stage eq "raw") or ($stage eq 'chip')) {
     1141                    $image->{astrom} = $astrom_file;
     1142                    $image->{cam_id} = $run->{cam_id};
     1143                    $image->{cam_path_base} = $run->{cam_path_base};
     1144                }
     1145                push @$results, $image;
     1146            }
     1147        }
     1148    } else {
     1149        &my_die ("sorry not done with $req_type\n", $PS_EXIT_PROG_ERROR);
     1150    }
     1151    return $results;
     1152}
     1153
     1154sub filterRuns {
     1155    my $stage      = shift;
     1156    my $need_magic = shift;
     1157    my $inputs     = shift;
     1158    my $verbose    = shift;
     1159
     1160    if ($inputs and (scalar @$inputs) == 1)  {
     1161        # one run nothing to do
     1162        return $inputs;
     1163    }
     1164
     1165    my $output = [];
     1166
     1167    return $output if (!$inputs or scalar @$inputs == 0);
     1168
     1169    my $id_name = $stage . "_id";
     1170
     1171    # input list is "order by exp_id, run_id DESC"   run_id is one of (chip_id, warp_id, diff_id)
     1172    print "Starting filterRuns\n";
     1173    my $last_exp_id = 0;
     1174    my $last_run_id = 0;
     1175    my $printed = 0;
     1176    foreach my $input (@$inputs) {
     1177        my $exp_id = $input->{exp_id};
     1178        my $run_id = $input->{$id_name};
     1179        my $magicked = $input->{magicked};  # this will be either stageRun.magicked or stage%file.magicked
     1180        my $state = $input->{state};
     1181
     1182        # can't process run in these states
     1183        if (($state eq 'new') or ($state eq 'purged') or ($state eq 'scrubbed')) {
     1184            print "skipping ${stage}Run $run_id for exp_id $exp_id in state $state\n";
     1185            next;
     1186        }
     1187
     1188        $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id));
     1189
     1190        # skip if we need magicked run and this one has never been magicked
     1191        if ($need_magic and !$magicked) {
     1192            print "skipping ${stage}Run $run_id for exp_id $exp_id not magicked\n" if !$printed;
     1193            next;
     1194        }
     1195
     1196        if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) {
     1197            print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed;
     1198            next;
     1199        }
     1200        print "Keeping ${stage}Run $run_id for $exp_id\n";
     1201        push @$output, $input;
     1202        $last_exp_id = $exp_id;
     1203        $last_run_id = $run_id;
     1204        $printed = 0;
     1205    }
     1206
     1207    my $num_runs = scalar @$output;
     1208    print "filterRuns returning $num_runs ${stage}Run\n";
     1209
     1210    return $output;
     1211}
     1212
     1213# run a command that produces metadata output and parse the results into an array of objects
     1214sub runToolAndParse {
     1215    my $command = shift;
     1216    my $verbose = shift;
     1217
     1218    my ($program) = split " ", $command;
     1219    $program = basename($program);
     1220
     1221    print "Running $command\n" if !$verbose;
     1222    my $start_tool = DateTime->now->mjd;
     1223    # run the command and parse the output
     1224
     1225    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1226                run(command => $command, verbose => $verbose);
     1227    unless ($success) {
     1228        # not sure if we should die here
     1229        print STDERR @$stderr_buf;
     1230        return undef;
     1231    }
     1232
     1233    my $now = DateTime->now->mjd;
     1234    my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.;
     1235    print "Time to run $program: $dtime_tool\n";
     1236
     1237    my $buf = join "", @$stdout_buf;
     1238    if (!$buf) {
     1239        return undef;
     1240    }
     1241
     1242    my $start_parse = DateTime->now->mjd;
     1243
     1244    my $mdcParser = PS::IPP::Metadata::Config->new;
     1245    my $results = parse_md_fast($mdcParser, $buf)
     1246        or die ("Unable to parse metadata config doc");
     1247
     1248    my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.;
     1249    print "Time to parse results from $program: $dtime_parse\n";
     1250
     1251    return $results;
     1252}
     1253
     1254sub findTools {
     1255    return if ($regtool);
     1256
     1257    my $missing_tools;
     1258    $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
     1259    $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
     1260    $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
     1261    $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
     1262    $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
     1263    $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
     1264    $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
     1265    $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords"
     1266                                                                and $missing_tools = 1);
     1267    $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
     1268    $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
     1269    if ($missing_tools) {
     1270        warn("Can't find required tools.");
     1271        exit ($PS_EXIT_CONFIG_ERROR);
     1272    }
     1273}
     1274
     1275# add a row_index array to a set of images with entries for each of the rows in a list
     1276sub setRowRefs {
     1277    my $rowList = shift;
     1278    my $images  = shift;
     1279
     1280    my $row_index = [];
     1281    for (my $i = 0; $i < scalar @$rowList; $i++) {
     1282        push @$row_index, $i;
     1283    }
     1284    foreach my $image (@$images) {
     1285        $image->{row_index} = $row_index;
     1286    }
     1287}
     1288
     1289sub my_die
     1290{
     1291    my $msg = shift;
     1292    my $fault = shift;
     1293
     1294    carp $msg;
     1295
     1296    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
     1297
     1298    return $fault;
     1299}
    104713001;
Note: See TracChangeset for help on using the changeset viewer.