Changeset 25027 for branches/pap/pstamp/scripts/pstampparse.pl

Timestamp:

Aug 7, 2009, 4:08:25 PM (17 years ago)

Author:

Paul Price

Message:

Merging trunk (r25026) to get up-to-date on old branch.

Location:

branches/pap

Files:

• . (modified) (1 prop)
• pstamp/scripts/pstampparse.pl (modified) (8 diffs)

branches/pap/pstamp/scripts/pstampparse.pl

@@ -13 +13 @@
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use PS::IPP::PStamp::Job qw( :standard );
+use File::Temp qw(tempfile);
 
 my $verbose;
 my $dbname;
+my $dbserver;
 my $req_id;
 my $request_file_name;
@@ -21 +23 @@
 my $out_dir;
 my $product;
+my $save_temps;
+my $no_update;
 
 GetOptions(
@@ -29 +33 @@
     'mode=s'    =>  \$mode,
     'dbname=s'  =>  \$dbname,
+    'dbserver=s'=>  \$dbserver,
     'verbose'   =>  \$verbose,
+    'save-temps'=>  \$save_temps,
+    'no-update' =>  \$no_update,
 );
 
@@ -54 +61 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
+my $dvoImagesAtCoords  = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
 my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
@@ -60 +68 @@
     exit ($PS_EXIT_CONFIG_ERROR);
 }
+
+$no_update = 1 if $mode eq "list_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -85 +95 @@
 die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
 
-if ($req_id) {
+if ($req_id and !$no_update) {
     my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
     $command .= " -outProduct $product";
     $command .= " -dbname $dbname" if $dbname;
+    $command .= " -dbserver $dbserver" if $dbserver;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -114 +125 @@
 }
 
+my @rowList;
 my $num_jobs = 0;
+my $imageList;
+my $stage;
+my $need_magic;
 foreach my $row (@$rows) {
+    # XXX: TODO: sanity check all parameters
+
+    # XXX: TODO: We shouldn't really just die in this loop.
+    # If we encounter an error for a particular row
+    # add a job with the proper fault code. If there is a db or config error we should probably just
+    # trash the request.
+
     my $rownum   = $row->{ROWNUM};
     my $job_type = $row->{JOB_TYPE};
+    
+    # parameters that select the images of interest
     my $project  = $row->{PROJECT};
     my $req_type = $row->{REQ_TYPE};
-    my $img_type = $row->{IMG_TYPE};
+    $stage = $row->{IMG_TYPE};
     my $id       = $row->{ID};
-    my $class_id = $row->{CLASS_ID};
+    my $component = $row->{COMPONENT};
+
     my $filter   = $row->{REQFILT};
     my $mjd_min = $row->{MJD_MIN};
     my $mjd_max = $row->{MJD_MAX};
-    my $x = $row->{CENTER_X};
-    my $y = $row->{CENTER_Y};
-    my $w = $row->{WIDTH};
-    my $h = $row->{HEIGHT};
-    my $coord_mask = $row->{COORD_MASK};
+
     my $option_mask= $row->{OPTION_MASK};
 
-    $class_id = "" if ($class_id eq "null" or $class_id eq "all");
+    die "valid region of interest is required to make postage stamps"
+        if (($job_type eq "stamp") and ! validROI($row));
+
+    my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+    $component = "" if (defined($component) and ($component eq "null" or $component eq "all"));
     
-    # XXX: TODO: sanity check all parameters
-
-    # XXX: TODO: We shouldn't just die in this loop.
-    # If we encounter an error for a particular row
-    # add a job with the proper fault code. If there is a db or config error we should probably just
-    # trash the request.
     die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
 
-    my $proj_hash = resolve_project($ipprc, $project, $dbname);
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
     die "project $project not found\n" unless $proj_hash;
 
-    my $image_db = $proj_hash->{dbname};
-    my $camera = $proj_hash->{camera};
-    my $need_magic = $proj_hash->{need_magic};
-
-    my $roi_string;
-    # XXX we're depending on other code to insure valid values for roi components
-    # this is checked in and ppstamp but I should check here so that we don't get that far,
-    # but not today
-    if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
-        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
-            $roi_string = "-pixcenter $x $y";
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+    $need_magic = $proj_hash->{need_magic};
+
+    # collect rows with the same images of interest in a list so that they
+    # can be processed optimially
+    if (@rowList) {
+        my $firstRow = $rowList[0];
+        # XXX: the collecting might work with !$skycenter but I need to think about it
+        if ($skycenter and same_images_of_interest($row, $firstRow)) {
+            push @rowList, $row;
+            # On to the next row
+            next;
         } else {
-            $roi_string = "-skycenter $x $y";
-        }
-        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
-            $roi_string .= " -pixrange $w $h";
-        } else {
-            $roi_string .= " -arcrange $w $h";
-        }
-    }
-
-    die "region of interest is required to make postage stamps"
-        if (($job_type eq "stamp") && !defined($roi_string));
-
-    # find the uris for this request
+            # queue the jobs for this set of rows
+            $num_jobs += queueJobs($mode, \@rowList, $imageList);
+            @rowList = ();
+        }
+    }
 
     if ($req_type eq "bycoord") {
-        die "region of interest is required for request type bycoord" if !defined($roi_string) ;
-        die "center must be specified in sky coordintes for bycoord" 
-            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS);
-    }
-
-    # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this request specification
-    my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id,
-            $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
-
-    if (!$images) {
+        die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter);
+        die "lookup bycoord is not yet implemented";
+    } else {
+
+        # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this
+        # request specification. An array reference is returned
+        my ($x, $y);
+        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $component, $skycenter,
+                $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+    }
+
+    if (!$imageList or !@$imageList) {
         print STDERR "no matching images found for row $rownum\n";
         next;
     }
-    if ($mode eq "list_uri") {
-        foreach my $image (@$images) {
-            print "$image->{image}\n";
-        }
-    } elsif ($job_type eq "get_image") {
-        # XXX TODO: Get rid of this block and use the same code as the stamp jobs
-        # XXX This doesn't work to get the mask and weight images
-        my $listfile = "$out_dir/filelist";
-
-
-        # map our img_type to the Data Store file types.
-        my %filelist_img_types = ( "raw" => "chip", 
-                                   "chip" => "chipproc",
-                                   "warp" => "warp", 
-                                   "stack"=>"stack",
-                                   "diff" => "diff");
-
-        my $filelist_img_type = $filelist_img_types{$img_type};
-
-        open LISTFILE, ">$listfile"
-            or die "failed to open file list: $listfile while parsing get_image request $req_id";
-
-        foreach my $image (@$images) {
-            my $class_id = $image->{class_id} ? $image->{class_id} : "";
-            print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n";
-        }
-        close LISTFILE;
-        $num_jobs++;
-        my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image"
-                    . " -uri $listfile -outputBase $out_dir -rownum $rownum";
-        $command .= " -dbname $dbname" if $dbname;
-        if ($mode eq "list_job") {
-            print "$command\n";
-        } else {
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                print STDERR @$stderr_buf;
-                # XXX TODO: now what? Should we mark the error state for the request?
-                die "failed to queue job for request $req_id: rownum: $rownum";
-            }
-        }
-    } else {
-        # sequence number for the the job for a request spec. Not to be confused with job_id
-        my $job_num = 0;
-
-        # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images
-        # use -list instead of -file
-        foreach my $image (@$images) {
-            my $uri = $image->{image};
-            my $exp_id = $image->{exp_id};
-            
-            my $args = $roi_string ? $roi_string : "";
-            $args .= " -class_id $class_id" if $class_id;
-
-            $job_num++;
-
-            my $output_base = "$out_dir/${rownum}_${job_num}";
-
-            # add astrometry file for raw and chip images if one is available
-            if (($img_type eq "chip") || ($img_type eq "raw")) {
-                $args .= " -astrom $image->{astrom}" if $image->{astrom};
-            }
-
-            if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
-                $args .= " -mask $image->{mask}";
-            }
-            if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
-                $args .= " -weight $image->{weight}";
-            }
-
-            # XXX sounds like magic will be handled outside of the postage stamp server
-            #if ($need_magic) {
-            #    die "no magic mask available" if ! $image->{magic_mask};
-            #    $args .= " -magic_mask $image->{magic_mask}" 
-            #}
-
-            # XXX: TODO: here is where we need to check whether or not the source inputs still exist
-            # and if not, queue a regeneration job and set the job state appropriately.
-
-            my $state = "run";
-
-            $num_jobs++;
-            my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type"
-                . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum"
-                . " -state $state";
-            $command .= " -exp_id $exp_id" if $exp_id;
-            $command .= " -dbname $dbname" if $dbname;
-
-            if ($mode eq "list_job") { 
-                # this is a debugging mode, just print the pstamptool that would have run
-                # this is sort of like the mode -noupdate that some other tools support
-                print "$command\n";
-            } else {
-                # mode eq "queue_job"
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    print STDERR @$stderr_buf;
-                    # XXX TODO: now what? Should we mark the error state for the request?
-                    # should we keep going for other uris? If so how do we report that some
-                    # of the work that the request wanted isn't going to get done
-                    die "failed to queue job for request $req_id";
-                }
-            }
-        }
-    }
+    $row->{need_magic} = $need_magic;
+    push @rowList, $row;
+}
+
+if (@rowList) {
+    $num_jobs += queueJobs($mode, \@rowList, $imageList);
 }
 
@@ -306 +223 @@
     exit 0;
 }
+
+
+sub queueJobsForRow
+{
+    my $row = shift;
+    my $stage = shift;
+    my $imageList = shift;
+    my $have_skycells = shift;
+    my $need_magic = shift;
+
+    # loop over images
+    my $job_num = 0;
+    foreach my $image (@$imageList) {
+        my $component;
+        if ($have_skycells) {
+            $component = $image->{skycell_id};
+        } else {
+            $component = $image->{class_id};
+        }
+
+        # skip this component if it is not in the list for this row
+        next if !$row->{components}->{$component};
+
+        my $rownum = $row->{ROWNUM};
+
+        my $roi_string;
+
+        # note values were insured to be numbers in validROI()
+        my $x = $row->{CENTER_X};
+        my $y = $row->{CENTER_Y};
+        my $w = $row->{WIDTH};
+        my $h = $row->{HEIGHT};
+        my $coord_mask = $row->{COORD_MASK};
+        if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
+            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
+                $roi_string = "-pixcenter $x $y";
+            } else {
+                $roi_string = "-skycenter $x $y";
+            }
+            if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
+                $roi_string .= " -pixrange $w $h";
+            } else {
+                $roi_string .= " -arcrange $w $h";
+            }
+        }
+
+        my $imagefile = $image->{image};
+        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
+            # XXX: should we add a faulted job so the client can know what happened?
+            print STDERR "skippping non-magicked image $imagefile\n" if $verbose;
+            next;
+        }
+        my $exp_id = $image->{exp_id};
+            
+        my $args = $roi_string ? $roi_string : "";
+        if ($stage eq "raw" or $stage eq "chip") {
+            $args .= " -class_id $component" if $component;
+        }
+
+        $job_num++;
+
+        # add astrometry file for raw and chip images if one is available
+        if (($stage eq "chip") || ($stage eq "raw")) {
+            $args .= " -astrom $image->{astrom}" if $image->{astrom};
+        }
+
+        $args .= " -file $imagefile";
+
+        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
+            $args .= " -mask $image->{mask}";
+        }
+        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
+            $args .= " -variance $image->{weight}";
+        }
+
+        my $output_base = "$out_dir/${rownum}_${job_num}";
+        my $argslist = "${output_base}.args";
+
+        # copy the argument list to a file
+        open ARGSLIST, ">$argslist" or die "failed to open $argslist";
+        print ARGSLIST "$args\n";
+        close ARGSLIST or die "failed to close $argslist";
+
+        # XXX: TODO: here is where we need to check whether or not the source inputs still exist
+        # and if not, queue an update job and set the job state appropriately.
+
+        my $newState = "run";
+
+        $num_jobs++;
+        my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
+                        . " -outputBase $output_base -rownum $rownum -state $newState";
+        $command .= " -exp_id $exp_id" if $exp_id;
+        $command .= " -dbname $dbname" if $dbname;
+        $command .= " -dbserver $dbserver" if $dbserver;
+
+        if ($mode eq "list_job") { 
+            # this is a debugging mode, just print the pstamptool that would have run
+            # this is sort of like the mode -noupdate that some other tools support
+            print "$command\n";
+        } elsif (!$no_update) {
+            # mode eq "queue_job"
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR @$stderr_buf;
+                # XXX TODO: now what? Should we mark the error state for the request?
+                # should we keep going for other uris? If so how do we report that some
+                # of the work that the request wanted isn't going to get done
+                die "failed to queue job for request $req_id";
+            }
+        } else {
+            print "skipping command: $command\n";
+        }
+    }
+    return $num_jobs;
+}
+
+sub queueJobs
+{
+    my $mode = shift;
+    my $rowList = shift;
+    my $imageList = shift;
+
+    my $firstRow = $rowList[0];
+    my $stage    = $firstRow->{IMG_TYPE};
+    my $job_type = $firstRow->{JOB_TYPE};
+    my $need_magic = $firstRow->{need_magic};
+
+    my $num_jobs = 0;
+
+    if ($mode eq "list_uri") {
+        foreach my $image (@$imageList) {
+            print "$image->{image}\n";
+        }
+    } elsif ($job_type eq "get_image") {
+        die "get_image jobs not implemented yet";
+    } else {
+        my $thisRun;
+
+        my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
+        foreach my $row (@$rowList) {
+            print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
+            $row->{components} = {};
+        }
+        close $pointsList;
+
+        my $have_skycells;
+        if (($stage eq "raw") or ($stage eq "chip")) {
+            $have_skycells = 0;
+        } else {
+            $have_skycells = 1;
+        }
+        
+        my $tess_dir_abs;
+        my $last_tess_id = "";
+        while ($thisRun = getOneRun($stage, $imageList)) {
+            {
+                my $command = "$dvoImagesAtCoords $pointsListName";
+                if ($have_skycells) {
+                    my $tess_id = $thisRun->[0]->{tess_id};
+                    if ($tess_id ne $last_tess_id) {
+                        $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id );
+                        $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs );
+                        $last_tess_id = $tess_id;
+                    }
+                    $command .= " -D CATDIR $tess_dir_abs";
+                } else {
+                    my $astrom = $thisRun->[0]->{astrom};
+                    die "no astrometry file found" if !$astrom;
+                    my $astrom_resolved = $ipprc->file_resolve($astrom);
+                    $command .= " -astrom $astrom_resolved";
+                }
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    print STDERR @$stderr_buf;
+                    my $rc = $error_code >> 8;
+                    die "dvoImagesAtCoords failed: $rc";
+                }
+                # now we have a list of row numbers and components
+                # eventually we might want to multiple stamp requests for the same image
+                # into the same ppstamp job but not yet. For now we will queue a new
+                my @lines = split "\n", join "", @$stdout_buf;
+                foreach my $line (@lines) {
+                    # parse the line, ignoring the ra and dec
+                    my ($rownum, undef, undef, $component) = split " ", $line;
+
+                    # I guess since we need this function we should be useing a hash for rowList 
+                    my $row = findRow($rownum, $rowList);
+                    $row->{components}->{$component} = 1;
+                }
+            }
+            
+            foreach my $row (@$rowList) {
+                $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic);
+            }
+        }
+    }
+    return $num_jobs;
+}
+
+sub get_run_id
+{
+    my $stage = shift;
+    my $image = shift;
+
+    if ($stage eq "raw") {
+        return $image->{exp_id};
+    } elsif ($stage eq "chip") {
+        return $image->{chip_id};
+    } elsif ($stage eq "warp") {
+        return $image->{warp_id};
+    } elsif ($stage eq "stack") {
+        return $image->{stack_id};
+    } elsif ($stage eq "diff") {
+        return $image->{diff_id};
+    } else {
+        die "unenexpected stage: $stage found";
+    }
+}
+
+# extract components from the imageList that have the same run id and return the list
+sub getOneRun {
+    my $stage = shift;
+    my $imageList = shift;
+
+    # return if array is empty
+    return undef if ! @$imageList;
+
+    my $last_run_id = 0;
+    my @runList;
+    while ($imageList->[0]) {
+        my $run_id = get_run_id($stage, $imageList->[0]);
+
+        last if ($last_run_id and ($run_id ne $last_run_id));
+
+        my $image = shift @$imageList;
+        $image->{stage} = $stage;
+        push @runList, $image;
+        $last_run_id = $run_id;
+    }
+    return \@runList;
+}
+
+sub same_images_of_interest {
+    my $r1 = shift;
+    my $r2 = shift;
+
+    return 0 if ($r1->{PROJECT}  ne $r2->{PROJECT});
+    return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE});
+    return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE});
+    return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
+    return 0 if ($r1->{ID} ne $r2->{ID});
+
+    if (defined($r1->{COMPONENT})) {
+        return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT});
+    } elsif (defined($r2->{COMPONENT})) {
+        return 0;
+    }
+
+    return 1;
+}
+
+sub validNumber
+{
+    my $val = shift;
+
+    return $val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/;
+}
+
+sub validROI
+{
+    my $row = shift;
+    return 0 if !validNumber($row->{CENTER_X});
+    return 0 if !validNumber($row->{CENTER_Y});
+    return 0 if !validNumber($row->{WIDTH});
+    return 0 if !validNumber($row->{HEIGHT});
+
+    return 1;
+}
+
+sub findRow
+{
+    my $rownum = shift;
+    my $rowList = shift;
+
+    foreach my $row (@$rowList) {
+        return $row if $row->{ROWNUM} eq $rownum;
+    }
+    return undef;
+}