Changeset 38077

Timestamp:

Apr 1, 2015, 12:39:27 PM (11 years ago)

Author:

bills

Message:

Remove the "past its sell by date" default_data_groups concept. The list quickly beomes
stale. Users who want the best results need to use SURVEY/IPP_RELEASE to be sure
to get properly selected data.
Change to using releasetool for lookups in more instances by guessing/defaulting the
SURVEY when it isn't supplied based on tess_id and data_group. If no guess is found
we default to 3PI for bycoord and byskycell requests.
"byexp" requests still fall back to not using releasetool if survey is not supplied
because the release tables sometimes become stale if duplicate camRuns get queued
for an exposure. We need MOPS' directly submitted requests to succeed. This can
be changed once camera run queuing is hardened to fail if a duplicate is attempted to
be queued.

File:

• trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (12 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -83 +83 @@
            ($req_type ne "byskycell");
 
+if (0) {
     if (!$data_group and !$default_data_groups) {
         $default_data_groups = load_data_groups($verbose);
     }
+}
 
     my $dateobs_begin;
@@ -211 +213 @@
     my $mjd_max   = $row->{MJD_MAX};
     my $data_group = $row->{DATA_GROUP};
+
+    # default_data_groups is a concept far past it's sell by date
+if (0) {
     if (isnull($data_group) and !$default_data_groups) {
         $default_data_groups = load_data_groups($verbose);
     }
+}
 
     my $rownum     = $row->{ROWNUM};
@@ -273 +279 @@
     my ($release_name, $survey, $default_tess_id) = get_release_info($row);
     if (!$tess_id and $default_tess_id) {
-        $tess_id = $default_tess_id;
+        # Don't want to set default tess_id in this context
+        # byid: no the tess_id is defined by the run
+        # byexp warp: This is the one case where it might make sense to use a default to avoid double lookup.
+        #   this could be avoided if we did a two level lookup in that case. Find the warpRun first then use the 
+        #   run's tess id to find the specific skycell(s).
+        # bydiff: no tess_id defined by the diff
+        # byskycell requires tess_id so this does not apply
+        # note: currently get_release_info only sets default_tess_id for byskycell and bycoord requests
+        if (ref($default_tess_id) eq 'ARRAY') {
+            $tess_id = $default_tess_id->[0];
+        } else {
+            $tess_id = $default_tess_id;
+        }
     }
 
@@ -393 +411 @@
     my $release_args;
     if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) {
-        if ($release_name or $survey) {
+        # XXX: Think hard before committing the change to use releasetool for byexp requests regardless.
+        # XXX: using releasetool by default for byexp requests will break for nightly data when 
+        # the release table survey task breaks.
+        # (MOPS one off requests for last night's data)
+        # we could fall back and do the old style lookup in that case.
+        # Need to fix the duplicate camRun problem.
+        my $force_releasetool_for_byexp = 0;
+        if ($release_name or $survey or ($force_releasetool_for_byexp and ($req_type eq 'byexp'))) {
             $use_releasetool = 1;
             if ($release_name) {
@@ -599 +624 @@
         $command .= " -data_group $data_group";
     } elsif (!$use_releasetool and $req_type eq 'byskycell') {
-        # XXX: Why am I using default data_groups only for byskycell requests?
+        # Default data_groups are used by bycoord in the camera run lookup and
+        # here for byskycell.
         $command .= $default_data_groups;
     }
@@ -773 +799 @@
         my ($release_name, $survey, $default_tess_id) = get_release_info($row);
         if (!$tess_id and $default_tess_id) {
-            $tess_id = $default_tess_id;
+            if (ref($default_tess_id) eq 'ARRAY') {
+                # XXX: handle the case where default_tess_id is an array
+                # For now just take the first entry (RINGS.V3)
+                $tess_id = $default_tess_id->[0];
+            } else {
+                $tess_id = $default_tess_id;
+            }
         }
 
@@ -1061 +1093 @@
     
 
+    my @tess_ids;
     my ($release_name, $survey, $default_tess_id) = get_release_info($row);
-    if (!$tess_id and $default_tess_id) {
-        $tess_id = $default_tess_id;
+    if ($tess_id) {
+        push @tess_ids, $tess_id;
+    } elsif ($default_tess_id) {
+        if (ref($default_tess_id) eq 'ARRAY') {
+            @tess_ids = @$default_tess_id;
+        } else {
+            push @tess_ids, $default_tess_id;
+        }
     }
 
     my $rowList = [$row];
 
-    my $results = ();
+    my $results = [];
     if (($stage eq "raw") or ($stage eq "chip")) {
 
@@ -1081 +1120 @@
                 next if $component and ($chip->{component} ne $component);
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id},
-                    $tess_id, $chip->{component}, $need_magic, 
+                    # lookup doesn't need tess_id in this context
+                    # $tess_id,
+                    undef, 
+                    $chip->{component}, $need_magic, 
                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
                     $fwhm_min, $fwhm_max,
@@ -1097 +1139 @@
                                               and ($stage ne "stack") and ($stage ne "diff");
 
-        my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
-
-        if (!$skycells or scalar @$skycells == 0) {
+        foreach my $tess_id (@tess_ids) {
+            my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
+
+            if ($skycells and scalar @$skycells != 0) {
+                # XXX: We are not applying the fwhm cuts unless we use releasetool
+                foreach my $skycell (@$skycells) {
+                    my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
+                        $skycell->{tess_id}, $skycell->{component}, $need_magic, 
+                        $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+                        $fwhm_min, $fwhm_max,
+                        undef, 0, $release_name, $survey,
+                        $verbose);
+
+                    next if !$these_results;
+                    push @$results, @$these_results;
+                }
+            }
+        }
+        if (scalar @$results == 0) {
             setErrorCodes($rowList, $PSTAMP_NO_IMAGE_MATCH);
-        } else {
-            # XXX: We are not applying the fwhm cuts unless we use releasetool
-            foreach my $skycell (@$skycells) {
-                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
-                    $skycell->{tess_id}, $skycell->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
-                    $fwhm_min, $fwhm_max,
-                    undef, 0, $release_name, $survey,
-                    $verbose);
-
-                next if !$these_results;
-                push @$results, @$these_results;
-            }
         }
     }
@@ -1976 +2021 @@
 }
 
+# get user supplied release paramters and the asssociated default_tess_id.
+# If release parameters are not supplied set defaults.
 sub get_release_info {
     my $row = shift;
@@ -1985 +2032 @@
         # no release check for survey
         $survey = isnull($row->{SURVEY_NAME}) ? "" : $row->{SURVEY_NAME};
-    } else {
-        # if release is given leave survey empty
-        $survey = "";
-    }
-
-    my $tess_id;
-    if ($release_name) {
-        # hack to optimize bycoord and byskycell lookups for well known releases
-        # XXX: todo query database and return a list for the survey or release
-        if ($release_name eq '3PI.PV1') {
-            $tess_id = 'RINGS.V3';
-        }
-    }
-
-    return ($release_name, $survey, $tess_id);
+    }
+
+    # XXX: From here down this function is currently very PS1 specific and depends on 
+    # conventions being used for release names.
+    # The comments following the lines  marked 'XXX to improve' give suggestions on
+    # how these hacks can be cleaned up. 
+
+    my $tess_id = $row->{TESS_ID};
+    if (!$release_name and !$survey) { 
+        # no release or survey. If user does not supply
+        # a data group use a default survey
+        my $req_type = $row->{REQ_TYPE};
+        if ($req_type eq "bycoord" or $req_type eq 'byskycell') {
+            # USING 3PI for bycoord and byskycell requests
+            # XXX to improve
+            # get default survey from project
+            my $data_group = $row->{DATA_GROUP};
+            if (isnull($data_group) and isnull($tess_id)) {
+                print "No release information supplied for this row. Defaulting to 3PI survey\n";
+                $survey = '3PI';
+            }
+        }
+    }
+
+    # if survey was not supplied, guess it from the release_name. This is used below
+    # to choose the default_tess_id
+    if (!$survey) {
+        if ($release_name) {
+            # XXX: to improve 
+            # get survey from ippRelease(release_name) in database. Would need to cache
+            # such results
+            if ($release_name =~ "^3PI") {
+                $survey = '3PI';
+            } elsif ($release_name =~ "^MD") {
+                $survey = substr($release_name, 0, 4);
+            } elsif ($release_name =~ "^SSS") {
+                $survey = 'SSS';
+            } elsif ($release_name =~ "^SAS") {
+                $survey = 'SAS';
+            } else {
+                # we don't know
+                print "Can't guess survey for $release_name\n";
+            }
+        } elsif (!isnull($tess_id)) {
+            # finally if it is a MD tessellation use the MD field as the survey
+            # (This will also work for M31 and STS when that data is released.
+            if ($tess_id =~ "^MD") {
+                $survey = substr($tess_id, 0, 4);
+            }
+        }
+    }
+
+    # choose default tess_id based on the survey
+
+    # Note: Setting a default_tess_id effectively disables lookup bycoord requests
+    # for previous tessellations unless the user supplies the tess_id (which is
+    # applied by the caller to this function).
+    # bills has decided that this is desirable behaviour since most everything has
+    # been processed on a V3 tess.
+    my $default_tess_id;
+    if ($survey) {
+        # XXX: to improve
+        # Put default tess_id in survey table.
+        # note users will need to supply tessellation to get at 3pi skycells
+        # in the cnp tessellation. Hmm. Maybe we need to be clever and look
+        # at declination or perhaps allow default_tess_id to be an array.
+        if ($survey eq '3PI') {
+            $default_tess_id = ['RINGS.V3', 'CNP.V3'];
+        } elsif ($survey eq 'SSS' or $survey eq 'SAS') {
+            $default_tess_id = 'RINGS.V3';
+        } elsif ($survey =~ 'MD') {
+            $default_tess_id = $survey . '.V3';
+        }
+    }
+            
+    # leave the returned survey blank if release_name was supplied
+    # XXX: Why? Answer: this is how this function always behaved and I'm not sure if the
+    # callers depend on that behavior.
+    $survey = "" if $release_name;
+
+    return ($release_name, $survey, $default_tess_id);
 }
 
@@ -2007 +2120 @@
     if (!isnull($run_type)) {
         $run_type = lc($run_type);
+        # XXX: these diff_mode values should be defined somewhere else.
         if ($run_type eq 'warp_warp') {
             $diff_mode = 1;