Changeset 35398

Timestamp:

Apr 12, 2013, 11:03:47 AM (13 years ago)

Author:

bills

Message:

If user specifies RELEASE_NAME or SURVEY use releasetool
to look up images.
When sources are selected extract them from the region of interest
using ppstamp. Use skycal cmf if available.

Location:

trunk

Files:

• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (28 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (4 diffs)
• pstamp/scripts/pstampparse.pl (modified) (9 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -15 +15 @@
 our @EXPORT_OK = qw( 
                     locate_images
-                    new_locate_images
+                    locate_images_for_row
                     resolve_project
                     getCamRunByCamID
@@ -79 +79 @@
            ($req_type ne "byskycell");
 
-    if (!$default_data_groups) {
+    if (!$data_group and !$default_data_groups) {
         $default_data_groups = load_data_groups($verbose);
     }
@@ -171 +171 @@
 
     my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
-        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+        undef, undef, undef, $verbose);
 
     return $results;
 }
 
-sub new_locate_images {
+sub locate_images_for_row {
     my $ipprc    = shift; 
     my $imagedb  = shift;
@@ -195 +196 @@
            ($req_type ne "byskycell");
 
-    if (!$default_data_groups) {
-        $default_data_groups = load_data_groups($verbose);
-    }
-
     my $stage     = $row->{IMG_TYPE};
     my $id        = $row->{ID};
@@ -208 +205 @@
     my $mjd_max   = $row->{MJD_MAX};
     my $data_group = $row->{DATA_GROUP};
+    if (isnull($data_group) and !$default_data_groups) {
+        $default_data_groups = load_data_groups($verbose);
+    }
 
     my $rownum     = $row->{ROWNUM};
@@ -230 +230 @@
         $filter = undef;
     }
-
-
 
     if ($req_type eq "bycoord") {
@@ -240 +238 @@
     }
 
-    # for compatability with the lower level functions
+    # for compatability with the lower level functions that haven't been converted to take a single row
     my $rowList = [$row];
 
@@ -251 +249 @@
         return $results;
     }
+
+    my ($release_name, $survey) = get_release_info($row);
 
     if ($req_type eq "bydiff") {
@@ -303 +303 @@
 
     my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
-        0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
+        undef, $release_name, $survey,
+        $verbose);
 
     return $results;
@@ -328 +330 @@
     my $mjd_min = shift;
     my $mjd_max = shift;
+    my $selectedAstrom = shift;
+    my $release_name = shift;
+    my $survey = shift;
     my $verbose  = shift;
 
@@ -352 +357 @@
     my $magic_arg = $need_magic ? " -destreaked" : "";
 
+    my $row = $rowList->[0];
+
     # all rows have the same center type
-    my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+    my $skycenter = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+
+    my $use_releasetool = 0;
+    my $release_args;
+    if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) {
+        unless (isnull($release_name) and isnull($survey)) {
+            $use_releasetool = 1;
+            $release_args = " -release_name $release_name" if $release_name;
+            $release_args .= " -surveyName $survey" if $survey;
+        }
+    }
 
     my $component_args;
@@ -384 +401 @@
         # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then
         # choose the chips containing the center pixel by calling selectComponets() below
-        if ($component or $use_imfile_id or !$skycenter) {
-            $command = "$chiptool -processedimfile -pstamp_order -dbname $imagedb";
-            if ($component and $component ne 'all') {
-                $class_id = $component;
-                $component_args = " -class_id $class_id";
-            }
-        } else {
-            $command = "$chiptool -listrun -pstamp_order -dbname $imagedb";
+        if ($use_releasetool) {
+            $command = "$releasetool -dbname $imagedb -priority_order -listrelexp $release_args";
             $choose_components = 1;
-        }
-        $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        } else {
+            $command = "$chiptool -pstamp_order -dbname $imagedb";
+            if ($component or $use_imfile_id or !$skycenter) {
+                $command .= " -processedimfile";
+                if ($component and $component ne 'all') {
+                    $class_id = $component;
+                    $component_args = " -class_id $class_id";
+                }
+            } else {
+                $command .= " -listrun";
+                $choose_components = 1;
+            }
+            $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        }
         # XXX: perhaps we should stop resolving images to this level here and do it at job run time.
         # With the mdc file it has all of the information that it needs.
@@ -408 +431 @@
         $want_astrom  = 1;
         $set_class_id = 1;
-    } elsif ($stage eq "warp") {
-        if ($component or $use_imfile_id or !$skycenter) {
-            $command = "$warptool -warped -pstamp_order -dbname $imagedb";
-            if ($component and $component ne 'all') {
-                $skycell_id = $component;
-                $component_args = " -skycell_id $skycell_id";
-            }
-        } else {
-            $command = "$warptool -listrun -pstamp_order -dbname $imagedb";
+    } elsif ($stage eq 'warp') {
+        if ($use_releasetool) {
+            $command = "$releasetool -dbname $imagedb -priority_order -listrelexp $release_args";
             $choose_components = 1;
-        }
-        $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
+        } else {
+            $command = "$warptool -pstamp_order -dbname $imagedb";
+            if ($component or $use_imfile_id or !$skycenter) {
+                $command .= " -warped";
+                if ($component and $component ne 'all') {
+                    $skycell_id = $component;
+                    $component_args = " -skycell_id $skycell_id";
+                }
+            } else {
+                $command = " -listrun";
+                $choose_components = 1;
+            }
+            $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
+        }
         $image_name   = "PSWARP.OUTPUT";
         $mask_name    = "PSWARP.OUTPUT.MASK";
@@ -427 +456 @@
         $base_name    = "path_base"; # name of the field for the warptool output
     } elsif ($stage eq "diff") {
+        # XXX: is this path ever used for diff images anymore?
         if ($component or $use_imfile_id or !$skycenter) {
             $command = "$difftool -diffskyfile -pstamp_order -dbname $imagedb";
@@ -447 +477 @@
     } elsif ($stage eq "stack") {
         $skycell_id = $component;
-        $command = "$stacktool -sumskyfile -dbname $imagedb";
-        # only consider stacks in full state.
-        $command .= " -state full";
-        $id_opt = "-stack_id";
+        if ($use_releasetool) {
+            $command = "$releasetool -listrelstack -dbname $imagedb $release_args";
+            $base_name = 'stack_path_base';
+        } else {
+            $command = "$stacktool -sumskyfile -dbname $imagedb";
+            # only consider stacks in full state.
+            $command .= " -state full";
+            $id_opt = "-stack_id";
+            $base_name = 'path_base';
+        }
         $component_args = " -skycell_id $skycell_id" if $skycell_id and $skycell_id ne  'all';
 
@@ -464 +500 @@
         # this is wrong but gets the right answer. Need to figure out how to find the
         # rule properly
-        #$cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
         $cmf_name     = "PSWARP.OUTPUT.SOURCES";
+        # $cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
         $psf_name    = "PPSTACK.TARGET.PSF";
-        $base_name   = "path_base";
     } else {
-        die "Unknown IMG_TYPe supplied: $stage";
+        die "Unknown IMG_TYPE supplied: $stage";
     }
 
@@ -512 +547 @@
     if (!isnull($data_group)) {
         $command .= " -data_group $data_group";
-    } elsif ($req_type eq 'byskycell') {
+    } elsif (!$use_releasetool and $req_type eq 'byskycell') {
+        # XXX: Why am I using default data_groups only for byskycell requests?
         $command .= $default_data_groups;
     }
@@ -527 +563 @@
         $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose);
     }
+
     if ($choose_components) {
         # the list of "images" is actually a list of "Runs"
         # match the coords in the rows to the components in the runs
         # returns an actual list of images
-        $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+        if ($skycenter) {
+            $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+        } else {
+            $images = selectComponentsByName($ipprc, $imagedb, $req_type, $stage, $component, $rowList, $images, $verbose);
+        }
     } else {
-        # put index for each of the rows into all of the the images
+        # put index for each of the rows into all of the the selected images
         setRowRefs($rowList, $images);
     }
@@ -597 +638 @@
             # dquery wants these set
             $out->{magicked} = 0;
-            $out->{data_state} = $out->{state};
+            if ($use_releasetool) {
+                if ($image->{skycal_path_base}) {
+                    $out->{astrom} = $image->{skycal_path_base} . ".cmf";
+                }
+                $out->{data_state} = $out->{state} = $out->{stack_state};
+            } else {
+                $out->{data_state} = $out->{state};
+            }
         }
         $out->{stage_id} = $stage_id;
@@ -605 +653 @@
         my $mask_base;
         if ($want_astrom) {
-            if (! defined $out->{astrom}) {
+            if ($selectedAstrom) {
+                $out->{astrom} = $selectedAstrom;
+                $out->{cam_path_base} = basename($selectedAstrom);
+            } elsif (! defined $out->{astrom}) {
                 if (! find_astrometry($ipprc, $imagedb, $out, $verbose)) {
                     print STDERR "failed to find astrometry for $stage $stage_id\n";
@@ -806 +857 @@
     my $mjd_min = shift;
     my $mjd_max = shift;
+    my $astrom = shift;
     my $verbose    = shift;
 
-    my ($release_name, $survey) = get_release($row);
+    my ($release_name, $survey) = get_release_info($row);
 
     my $rowList = [$row];
@@ -825 +877 @@
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id},
                     $tess_id, $chip->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
+                    $chip->{astrom}, undef, undef,
+                    $verbose);
 
                 next if !$these_results;
@@ -845 +899 @@
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
                     $skycell->{tess_id}, $skycell->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+                    undef, $release_name, $survey,
+                    $verbose);
 
                 next if !$these_results;
@@ -882 +938 @@
     my $using_camtool = 0;
     if ($release_name or $survey) {
-        my $command = "$releasetool -dbname $imagedb -listrelexp";
-        if ($survey) {
+        $command = "$releasetool -dbname $imagedb -listrelexp";
+        if (!isnull($survey)) {
             $command .= " -survey_name $survey";
         }
-        if ($release_name) {
+        if (!isnull($release_name)) {
             $command .= " -release_name $release_name";
         } else {
@@ -935 +991 @@
         if ($using_camtool) {
             next if $result->{fault};
-            updateCamRunCache($result);
-        }
+        } else {
+            $result->{path_base} = $result->{cam_path_base};
+        }
+        updateCamRunCache($result);
 
         $last_exp_id = $exp_id;
@@ -1008 +1066 @@
             cam_id    => $result->{cam_id},
             class_id  => $class_id,
-            component => $class_id
+            component => $class_id,
+            astrom    => $astrom
         };
         if ($stage eq "chip") {
@@ -1444 +1503 @@
 }
 
+sub selectComponentsByName {
+    my $ipprc = shift;
+    my $imagedb = shift;
+    my $req_type = shift;
+    my $stage  = shift;
+    my $component  = shift;
+    my $rowList = shift;
+    my $runList = shift;
+    my $verbose = shift;
+    my $results = [];
+
+    $component = "" if $component and lc($component) eq 'all';
+
+    # all runs match all rows so they can share a common row_index
+    # XXX: I'm not sure we ever actually get here with more than one
+    # row
+    my $numRows = scalar @$rowList;
+    my $row_index = [];
+    for (my $i=0; $i < $numRows; $i++) {
+        push @$row_index, $i;
+    }
+    
+    if (($req_type eq "byid") or ($req_type eq "byexp")) {
+        my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", "");
+        foreach my $run (@$runList) {
+            # now find the images for this run
+            #foreach my $c (keys %components) {
+            my $command;
+            my $stage_id;
+            if ($stage eq 'raw') {
+                $stage_id = $run->{exp_id};
+                $command = "$regtool -processedimfile -exp_id $stage_id";
+                $command .= " -class_id $component" if $component;
+            } elsif ($stage eq 'chip') {
+                $stage_id = $run->{chip_id};
+                $command = "$chiptool -processedimfile -chip_id $stage_id";
+                $command .= " -class_id $component" if $component;
+            } elsif ($stage eq 'warp') {
+                $stage_id = $run->{warp_id};
+                $command = "$warptool -warped -warp_id $stage_id";
+                $command .= " -skycell_id $component" if $component;
+            }
+            $command .= " -dbname $imagedb";
+            my $images = runToolAndParse($command, $verbose);
+            if (!defined $images) {
+                print "No  $component found for $stage ${stage}_id $stage_id\n";
+                next;
+            }
+            foreach my $image (@$images) {
+                $image->{row_index} = $row_index;
+                if (($stage eq "raw") or ($stage eq 'chip')) {
+                    $image->{cam_id} = $run->{cam_id};
+                    $image->{cam_path_base} = $run->{cam_path_base};
+                }
+                push @$results, $image;
+            }
+        }
+    } else {
+        &my_die ("selectComponentsByName not supported for REQ_TYPE: $req_type\n", $PS_EXIT_PROG_ERROR);
+    }
+    return $results;
+}
+
 sub filterRuns {
     my $stage      = shift;
@@ -1627 +1749 @@
 }
 
-sub get_release {
+sub get_release_info {
     my $row = shift;
 
     my $survey;
     my $release_name = $row->{IPP_RELEASE};
-    if (!isnull($release_name)) {
+    if (isnull($release_name)) {
+        $release_name = "";
         # no release check for survey
         $survey = $row->{SURVEY_NAME};
+    } else {
+        $survey = "";
     }
 

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -113 +113 @@
 
     if ($stage eq "raw") {
+        # zap options that don't apply to raw stage
         $options &= ~($PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE); 
     }
@@ -136 +137 @@
         $argString .= " -astrom $params->{astrom}";
         push @file_list, $params->{astrom};
+    }
+
+    if ($options & $PSTAMP_SELECT_SOURCES) {
+        # Extract sources from astrometry file if provided. This will be the smf for chip stage
+        # or the skycal cmf for stacks
+        if ($params->{astrom}) {
+            $argString .= " -write_cmf";
+            if ($stage eq 'stack') {
+                # Set psphot recipe to STACKPHOT so that the extended source paramters will
+                # be copied from the cmf file.
+                $argString .= " -recipe PSPHOT STACKPHOT"
+            }
+        } elsif ($params->{cmf}) {
+            $argString .= " -write_cmf";
+            push @file_list, $params->{cmf};
+        } else {
+            print "Could not find suitable sources file will not write cmf\n";
+        }
     }
 
@@ -254 +273 @@
                            $PSTAMP_SELECT_MASK     => "mk.fits",
                            $PSTAMP_SELECT_VARIANCE => "wt.fits",
+                           $PSTAMP_SELECT_SOURCES  => "cmf",
                            $PSTAMP_SELECT_JPEG     => "jpg");
 
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG);
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
 
         foreach my $key (keys (%extensions)) {
@@ -400 +420 @@
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $cmf_file;
-        if ($stage ne 'chip') {
-            # we don't ship chip stage cmf files because they may not be censored
-            $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
-        }
+#        if ($stage ne 'chip') {
+#            # we don't ship chip stage cmf files because they may not be censored
+#            $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
+#        }
 
         my $outdir = dirname($output_base);

trunk/pstamp/scripts/pstampparse.pl

@@ -27 +27 @@
 my $outdir;
 my $product;
-my $label;
+my $label = "";
 my $save_temps;
 my $no_update;
@@ -57 +57 @@
     die "outdir is required"  if !$outdir;
     die "product is required"  if !$product;
+} else {
+    $req_id = 0;
+    $outdir = "nowhere";
+    $product = "dummy";
 }
 
@@ -83 +87 @@
 $pstamptool .= " -dbserver $dbserver" if $dbserver;
 
-# list_job is a deugging mode
-$no_update = 1 if $mode eq "list_job";
+# If $mode is not queue_job we are using a debugging mode
+# do not update the database
+$no_update = 1 if $mode ne "queue_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 #
-# get the data from the extension header
+# Read the keywords from the extension header
 #
 my $fields_output;
@@ -96 +101 @@
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    # fields doesn't return zero when it succeeds
-    #unless ($success) {
-    #    print STDERR @$stderr_buf;
-    #}
+
+    # note fields doesn't return zero when it succeeds.
     $fields_output = join "", @$stdout_buf;
 }
@@ -105 +108 @@
 
 # make sure the file contains what we are expecting
-# This program shouldn't have been run if the request file is bogus.
-my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
-my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
-my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
+# pstamp_parser_run.pl would not have run this program unless the request file was ok
+my_die("$request_file_name does not contain EXTNAME\n", $PS_EXIT_PROG_ERROR) if !$extname;
+my_die("$request_file_name is not a PS1_PS_REQUEST\n", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
+my_die("REQ_NAME not found in $request_file_name\n", $PS_EXIT_PROG_ERROR)  if (!$req_name);
+my_die("wrong EXTVER $extver found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
 
 if ($extver >= 2) {
     # We have a version 2 file. Require that the new keywords be supplied. 
-    my_die("action not supplied in version $extver request file $request_file_name", $PSTAMP_INVALID_REQUEST) unless defined $action;
-
-    my_die("invalid action: $action supplied in version $extver request file $request_file_name", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
-
-    my_die("email not supplied in version $extver request file $request_file_name", $PSTAMP_INVALID_REQUEST) unless $email;
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
     # XXX check for "valid" $email
 } else {
@@ -437 +441 @@
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
-    if ($req_type eq 'bycoord' and $row->{MJD_MIN} == 0) {
+    if ($req_type eq 'bycoord' and $row->{IMG_TYPE} ne 'stack' and $row->{MJD_MIN} == 0) {
         # ... so unless the user sets mjd_min clamp it to 2009-04-01
         # XXX: This value should live in the pstampProject table not be hardcoded here
@@ -454 +458 @@
     print "\nCalling new_locate_images for row: $rownum\n";
 
-    $imageList = new_locate_images($ipprc, $image_db, $camera, $row, $verbose);
+    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
 
     my $dtime_locate = gettimeofday() - $start_locate;
@@ -736 +740 @@
 
     if ($mode eq "list_uri") {
+        $num_jobs = $imageList ? scalar @$imageList : 0;
+        print "List of $num_jobs Images selected for row: $firstRow->{ROWNUM}\n";
         foreach my $image (@$imageList) {
             print "$image->{image}\n";
+            ++$firstRow->{job_num};
         }
     } elsif ($job_type eq "get_image") {
@@ -1156 +1163 @@
     my $fault = shift;
 
-    carp $msg;
+    print STDERR $msg;
 
     # we don't fault the request here pstamp_parser_run.pl handles that if necessary