Changeset 24951 for branches/czw_branch/cleanup/pstamp/scripts/pstampparse.pl

Timestamp:

Jul 30, 2009, 5:20:29 PM (17 years ago)

Author:

watersc1

Message:

Finished up my edits to the detrend cleanup, and some changes to my
copy of burntool and the pslib astrometry. Detrend cleanup has not
been tested yet. That's up next.

Location:

branches/czw_branch/cleanup

Files:

• . (modified) (1 prop)
• pstamp/scripts/pstampparse.pl (modified) (9 diffs)

branches/czw_branch/cleanup/pstamp/scripts/pstampparse.pl

@@ -13 +13 @@
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use PS::IPP::PStamp::Job qw( :standard );
+use File::Temp qw(tempfile);
 
 my $verbose;
 my $dbname;
+my $dbserver;
 my $req_id;
 my $request_file_name;
@@ -21 +23 @@
 my $out_dir;
 my $product;
+my $save_temps;
+my $no_update;
 
 GetOptions(
@@ -29 +33 @@
     'mode=s'    =>  \$mode,
     'dbname=s'  =>  \$dbname,
+    'dbserver=s'=>  \$dbserver,
     'verbose'   =>  \$verbose,
+    'save-temps'=>  \$save_temps,
+    'no-update' =>  \$no_update,
 );
 
@@ -54 +61 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
+my $dvoImagesAtCoords  = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
 my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
@@ -60 +68 @@
     exit ($PS_EXIT_CONFIG_ERROR);
 }
+
+$no_update = 1 if $mode eq "list_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -85 +95 @@
 die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
 
-if ($req_id) {
+if ($req_id and !$no_update) {
     my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
     $command .= " -outProduct $product";
     $command .= " -dbname $dbname" if $dbname;
+    $command .= " -dbserver $dbserver" if $dbserver;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -114 +125 @@
 }
 
+my @rowList;
 my $num_jobs = 0;
+my $imageList;
+my $stage;
+my $need_magic;
 foreach my $row (@$rows) {
-    my $rownum   = $row->{ROWNUM};
-    my $job_type = $row->{JOB_TYPE};
-    my $project  = $row->{PROJECT};
-    my $req_type = $row->{REQ_TYPE};
-    my $img_type = $row->{IMG_TYPE};
-    my $id       = $row->{ID};
-    my $class_id = $row->{CLASS_ID};
-    my $filter   = $row->{REQFILT};
-    my $mjd_min = $row->{MJD_MIN};
-    my $mjd_max = $row->{MJD_MAX};
-    my $x = $row->{CENTER_X};
-    my $y = $row->{CENTER_Y};
-    my $w = $row->{WIDTH};
-    my $h = $row->{HEIGHT};
-    my $coord_mask = $row->{COORD_MASK};
-    my $option_mask= $row->{OPTION_MASK};
-
-    $class_id = "" if ($class_id eq "null" or $class_id eq "all");
-    
     # XXX: TODO: sanity check all parameters
 
@@ -141 +137 @@
     # add a job with the proper fault code. If there is a db or config error we should probably just
     # trash the request.
+
+    my $rownum   = $row->{ROWNUM};
+    my $job_type = $row->{JOB_TYPE};
+    
+    # parameters that select the images of interest
+    my $project  = $row->{PROJECT};
+    my $req_type = $row->{REQ_TYPE};
+    $stage = $row->{IMG_TYPE};
+    my $id       = $row->{ID};
+    my $class_id = $row->{CLASS_ID};
+    my $component = $row->{COMPONENT};
+    my $filter   = $row->{REQFILT};
+    my $mjd_min = $row->{MJD_MIN};
+    my $mjd_max = $row->{MJD_MAX};
+    my $option_mask= $row->{OPTION_MASK};
+
+    die "region of interest is required to make postage stamps"
+        if (($job_type eq "stamp") and ! validROI($row));
+
+    my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+    $class_id = "" if (defined($class_id) and ($class_id eq "null" or $class_id eq "all"));
+    $component = "" if (defined($component) and ($component eq "null" or $component eq "all"));
+    
     die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
 
-    my $proj_hash = resolve_project($ipprc, $project, $dbname);
+
+    # note: resolve_project remembers the last project returned so avoids running
+    # pstamptool every time
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
     die "project $project not found\n" unless $proj_hash;
 
-    my $image_db = $proj_hash->{dbname};
-    my $camera = $proj_hash->{camera};
-    my $need_magic = $proj_hash->{need_magic};
-
-    my $roi_string;
-    # XXX we're depending on other code to insure valid values for roi components
-    # this is checked in and ppstamp but I should check here so that we don't get that far,
-    # but not today
-    if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
-        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
-            $roi_string = "-pixcenter $x $y";
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+    $need_magic = $proj_hash->{need_magic};
+
+    # collect rows with the same images of interest in a list so that they
+    # can be processed optimially
+    if (@rowList) {
+        my $firstRow = $rowList[0];
+        if ($skycenter and same_images_of_interest($row, $firstRow)) {
+            push @rowList, $row;
+            # On to the next row
+            next;
         } else {
-            $roi_string = "-skycenter $x $y";
-        }
-        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
-            $roi_string .= " -pixrange $w $h";
-        } else {
-            $roi_string .= " -arcrange $w $h";
-        }
-    }
-
-    die "region of interest is required to make postage stamps"
-        if (($job_type eq "stamp") && !defined($roi_string));
-
-    # find the uris for this request
+            # Process the jobs for this set of rows
+            $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList);
+            @rowList = ();
+        }
+    }
 
     if ($req_type eq "bycoord") {
-        die "region of interest is required for request type bycoord" if !defined($roi_string) ;
-        die "center must be specified in sky coordintes for bycoord" 
-            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS);
-    }
-
-    # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this request specification
-    my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id,
-            $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
-
-    if (!$images) {
+        die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter);
+        die "lookup bycoord is not yet implemented";
+    } else {
+
+        # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this
+        # request specification. An array reference is returned
+        my ($x, $y);
+        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $class_id, $skycenter,
+                $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+    }
+
+    if (!$imageList or !@$imageList) {
         print STDERR "no matching images found for row $rownum\n";
         next;
     }
-    if ($mode eq "list_uri") {
-        foreach my $image (@$images) {
-            print "$image->{image}\n";
-        }
-    } elsif ($job_type eq "get_image") {
-        # XXX TODO: Get rid of this block and use the same code as the stamp jobs
-        # XXX This doesn't work to get the mask and weight images
-        my $listfile = "$out_dir/filelist";
-
-
-        # map our img_type to the Data Store file types.
-        my %filelist_img_types = ( "raw" => "chip", 
-                                   "chip" => "chipproc",
-                                   "warp" => "warp", 
-                                   "stack"=>"stack",
-                                   "diff" => "diff");
-
-        my $filelist_img_type = $filelist_img_types{$img_type};
-
-        open LISTFILE, ">$listfile"
-            or die "failed to open file list: $listfile while parsing get_image request $req_id";
-
-        foreach my $image (@$images) {
-            my $class_id = $image->{class_id} ? $image->{class_id} : "";
-            print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n";
-        }
-        close LISTFILE;
-        $num_jobs++;
-        my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image"
-                    . " -uri $listfile -outputBase $out_dir -rownum $rownum";
-        $command .= " -dbname $dbname" if $dbname;
-        if ($mode eq "list_job") {
-            print "$command\n";
-        } else {
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                print STDERR @$stderr_buf;
-                # XXX TODO: now what? Should we mark the error state for the request?
-                die "failed to queue job for request $req_id: rownum: $rownum";
-            }
-        }
-    } else {
-        # sequence number for the the job for a request spec. Not to be confused with job_id
-        my $job_num = 0;
-
-        # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images
-        # use -list instead of -file
-        foreach my $image (@$images) {
-            my $uri = $image->{image};
-            my $exp_id = $image->{exp_id};
-            
-            my $args = $roi_string ? $roi_string : "";
-            $args .= " -class_id $class_id" if $class_id;
-
-            $job_num++;
-
-            my $output_base = "$out_dir/${rownum}_${job_num}";
-
-            # add astrometry file for raw and chip images if one is available
-            if (($img_type eq "chip") || ($img_type eq "raw")) {
-                $args .= " -astrom $image->{astrom}" if $image->{astrom};
-            }
-
-            if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
-                $args .= " -mask $image->{mask}";
-            }
-            if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
-                $args .= " -weight $image->{weight}";
-            }
-
-            # XXX sounds like magic will be handled outside of the postage stamp server
-            #if ($need_magic) {
-            #    die "no magic mask available" if ! $image->{magic_mask};
-            #    $args .= " -magic_mask $image->{magic_mask}" 
-            #}
-
-            # XXX: TODO: here is where we need to check whether or not the source inputs still exist
-            # and if not, queue a regeneration job and set the job state appropriately.
-
-            my $state = "run";
-
-            $num_jobs++;
-            my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type"
-                . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum"
-                . " -state $state";
-            $command .= " -exp_id $exp_id" if $exp_id;
-            $command .= " -dbname $dbname" if $dbname;
-
-            if ($mode eq "list_job") { 
-                # this is a debugging mode, just print the pstamptool that would have run
-                # this is sort of like the mode -noupdate that some other tools support
-                print "$command\n";
-            } else {
-                # mode eq "queue_job"
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    print STDERR @$stderr_buf;
-                    # XXX TODO: now what? Should we mark the error state for the request?
-                    # should we keep going for other uris? If so how do we report that some
-                    # of the work that the request wanted isn't going to get done
-                    die "failed to queue job for request $req_id";
-                }
-            }
-        }
-    }
+    push @rowList, $row;
+}
+
+if (@rowList) {
+    $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList);
 }
 
@@ -306 +221 @@
     exit 0;
 }
+
+
+sub makeJobsForRow
+{
+    my $row = shift;
+    my $imageList = shift;
+    my $have_skycells = shift;
+    my $need_magic = shift;
+
+    # loop over images
+    my $job_num = 0;
+    foreach my $image (@$imageList) {
+        my $component;
+        my $class_id;   # get rid of this use componet
+        if ($have_skycells) {
+            $component = $image->{skycell_id};
+        } else {
+            $component = $image->{class_id};
+            $class_id = $component;
+        }
+
+        # skip this component if it is not in the list for this row
+        next if !$row->{components}->{$component};
+
+        my $rownum = $row->{ROWNUM};
+
+        my $roi_string;
+
+        # XXX we're depending on other code to insure valid values for roi components
+        # this is checked in and ppstamp but I should check here so that we don't get that far,
+        # but not today
+        my $x = $row->{CENTER_X};
+        my $y = $row->{CENTER_Y};
+        my $w = $row->{WIDTH};
+        my $h = $row->{HEIGHT};
+        my $coord_mask = $row->{COORD_MASK};
+        if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
+            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
+                $roi_string = "-pixcenter $x $y";
+            } else {
+                $roi_string = "-skycenter $x $y";
+            }
+            if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
+                $roi_string .= " -pixrange $w $h";
+            } else {
+                $roi_string .= " -arcrange $w $h";
+            }
+        }
+
+        my $uri = $image->{image};
+        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
+            print STDERR "skippping non-magicked image $uri\n" if $verbose;
+            next;
+        }
+        my $exp_id = $image->{exp_id};
+            
+        my $args = $roi_string ? $roi_string : "";
+        $args .= " -class_id $class_id" if $class_id;
+
+        $job_num++;
+
+        my $output_base = "$out_dir/${rownum}_${job_num}";
+
+        # add astrometry file for raw and chip images if one is available
+        if (($stage eq "chip") || ($stage eq "raw")) {
+            $args .= " -astrom $image->{astrom}" if $image->{astrom};
+        }
+
+        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
+            $args .= " -mask $image->{mask}";
+        }
+        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
+            $args .= " -weight $image->{weight}";
+        }
+
+        # XXX: TODO: here is where we need to check whether or not the source inputs still exist
+        # and if not, queue an update job and set the job state appropriately.
+
+        my $newState = "run";
+
+        $num_jobs++;
+        my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}"
+            . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum"
+            . " -state $newState";
+        $command .= " -exp_id $exp_id" if $exp_id;
+        $command .= " -dbname $dbname" if $dbname;
+        $command .= " -dbserver $dbserver" if $dbserver;
+
+        if ($mode eq "list_job") { 
+            # this is a debugging mode, just print the pstamptool that would have run
+            # this is sort of like the mode -noupdate that some other tools support
+            print "$command\n";
+        } else {
+            # mode eq "queue_job"
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR @$stderr_buf;
+                # XXX TODO: now what? Should we mark the error state for the request?
+                # should we keep going for other uris? If so how do we report that some
+                # of the work that the request wanted isn't going to get done
+                die "failed to queue job for request $req_id";
+            }
+        }
+    }
+    return $num_jobs;
+}
+
+sub makeJobs
+{
+    my $mode = shift;
+    my $stage = shift;
+    my $need_magic = shift;
+    my $rowList = shift;
+    my $imageList = shift;
+
+    my $firstRow = $rowList[0];
+    my $job_type = $firstRow->{JOB_TYPE};
+
+    my $num_jobs = 0;
+
+    if ($mode eq "list_uri") {
+        foreach my $image (@$imageList) {
+            print "$image->{image}\n";
+        }
+    } elsif ($job_type eq "get_image") {
+        die "get_image jobs not implemented yet";
+    } else {
+        my $thisRun;
+
+        my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
+        foreach my $row (@$rowList) {
+            print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
+            $row->{components} = {};
+        }
+        close $pointsList;
+
+        my $have_skycells;
+        if (($stage eq "raw") or ($stage eq "chip")) {
+            $have_skycells = 0;
+        } else {
+            $have_skycells = 1;
+        }
+        
+        my $tess_dir_abs;
+        my $last_tess_id = "";
+        while ($thisRun = getOneRun($stage, $imageList)) {
+            {
+                my $command = "$dvoImagesAtCoords $pointsListName";
+                if ($have_skycells) {
+                    my $tess_id = $thisRun->[0]->{tess_id};
+                    if ($tess_id ne $last_tess_id) {
+                        $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id );
+                        $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs );
+                        $last_tess_id = $tess_id;
+                    }
+                    $command .= " -D CATDIR $tess_dir_abs";
+                } else {
+                    my $astrom = $thisRun->[0]->{astrom};
+                    die "no astrometry file found" if !$astrom;
+                    $command .= " -astrom $astrom";
+                }
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    print STDERR @$stderr_buf;
+                    my $rc = $error_code >> 8;
+                    die "dvoImagesAtCoords failed: $rc";
+                }
+                # now we have a list of row numbers and components
+                # eventually we might want to multiple stamp requests for the same image
+                # into the same ppstamp job but not yet. For now we will queue a new
+                my @lines = split "\n", join "", @$stdout_buf;
+                foreach my $line (@lines) {
+                    # parse the line, ignoring the ra and dec
+                    my ($rownum, undef, undef, $component) = split " ", $line;
+
+                    # I guess since we need this function we should be useing a hash for rowList 
+                    my $row = findRow($rownum, $rowList);
+                    $row->{components}->{$component} = 1;
+                }
+            }
+            
+
+                # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images
+                # use -list instead of -file
+            foreach my $row (@$rowList) {
+                $num_jobs += makeJobsForRow($row, $thisRun, $have_skycells, $need_magic);
+            }
+        }
+    }
+    return $num_jobs;
+}
+
+sub get_run_id
+{
+    my $stage = shift;
+    my $image = shift;
+
+    if ($stage eq "raw") {
+        return $image->{exp_id};
+    } elsif ($stage eq "chip") {
+        return $image->{chip_id};
+    } elsif ($stage eq "warp") {
+        return $image->{warp_id};
+    } elsif ($stage eq "stack") {
+        return $image->{stack_id};
+    } elsif ($stage eq "diff") {
+        return $image->{diff_id};
+    } else {
+        die "unenexpected stage: $stage found";
+    }
+}
+
+# extract components from the imageList that have the same run id and return the list
+sub getOneRun {
+    my $stage = shift;
+    my $imageList = shift;
+
+    # return if array is empty
+    return undef if ! @$imageList;
+
+    my $last_run_id = 0;
+    my @runList;
+    while ($imageList->[0]) {
+        my $run_id = get_run_id($stage, $imageList->[0]);
+
+        last if ($last_run_id and ($run_id ne $last_run_id));
+
+        my $image = shift @$imageList;
+        push @runList, $image;
+        $last_run_id = $run_id;
+    }
+    return \@runList;
+}
+
+sub same_images_of_interest {
+    my $r1 = shift;
+    my $r2 = shift;
+
+    return 0 if ($r1->{project} ne $r2->{project});
+    return 0 if ($r1->{job_type} ne $r2->{job_type});
+    return 0 if ($r1->{req_type} ne $r2->{req_type});
+    return 0 if ($r1->{img_type} ne $r2->{img_type});
+    return 0 if ($r1->{id} ne $r2->{id});
+#    XXX: turn this on when we change the request file format to change class_id into component
+#    return false if ($r1->{component} ne $r2->{component});
+
+    return 1;
+}
+
+sub validROI
+{
+    my $row = shift;
+    return 0 if !defined $row->{CENTER_X};
+    return 0 if !defined $row->{CENTER_Y};
+    return 0 if !defined $row->{WIDTH};
+    return 0 if !defined $row->{HEIGHT};
+
+    return 1;
+}
+
+sub findRow
+{
+    my $rownum = shift;
+    my $rowList = shift;
+
+    foreach my $row (@$rowList) {
+        return $row if $row->{ROWNUM} eq $rownum;
+    }
+    return undef;
+}