Changeset 38119 for tags/ipp-20150329

Timestamp:

Apr 13, 2015, 6:12:49 PM (11 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

tags/ipp-20150329

Files:

• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (12 diffs)
• dbconfig/changes.txt (modified) (1 diff)
• ppSkycell/src/ppSkycellLoop.c (modified) (1 diff)
• pstamp/scripts/pstamp_job_run.pl (modified) (3 diffs)
• tools/checkexp (modified) (5 diffs)
• tools/mysql-dump/delete_log_spacing.sh (copied) (copied from trunk/tools/mysql-dump/delete_log_spacing.sh )
• tools/mysql-dump/gpc1_install.sh (modified) (2 diffs)

tags/ipp-20150329/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -83 +83 @@
            ($req_type ne "byskycell");
 
+if (0) {
     if (!$data_group and !$default_data_groups) {
         $default_data_groups = load_data_groups($verbose);
     }
+}
 
     my $dateobs_begin;
@@ -211 +213 @@
     my $mjd_max   = $row->{MJD_MAX};
     my $data_group = $row->{DATA_GROUP};
+
+    # default_data_groups is a concept far past it's sell by date
+if (0) {
     if (isnull($data_group) and !$default_data_groups) {
         $default_data_groups = load_data_groups($verbose);
     }
+}
 
     my $rownum     = $row->{ROWNUM};
@@ -273 +279 @@
     my ($release_name, $survey, $default_tess_id) = get_release_info($row);
     if (!$tess_id and $default_tess_id) {
-        $tess_id = $default_tess_id;
+        # Don't want to set default tess_id in this context
+        # byid: no the tess_id is defined by the run
+        # byexp warp: This is the one case where it might make sense to use a default to avoid double lookup.
+        #   this could be avoided if we did a two level lookup in that case. Find the warpRun first then use the 
+        #   run's tess id to find the specific skycell(s).
+        # bydiff: no tess_id defined by the diff
+        # byskycell requires tess_id so this does not apply
+        # note: currently get_release_info only sets default_tess_id for byskycell and bycoord requests
+        if (ref($default_tess_id) eq 'ARRAY') {
+            $tess_id = $default_tess_id->[0];
+        } else {
+            $tess_id = $default_tess_id;
+        }
     }
 
@@ -393 +411 @@
     my $release_args;
     if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) {
-        if ($release_name or $survey) {
+        # XXX: Think hard before committing the change to use releasetool for byexp requests regardless.
+        # XXX: using releasetool by default for byexp requests will break for nightly data when 
+        # the release table survey task breaks.
+        # (MOPS one off requests for last night's data)
+        # we could fall back and do the old style lookup in that case.
+        # Need to fix the duplicate camRun problem.
+        my $force_releasetool_for_byexp = 0;
+        if ($release_name or $survey or ($force_releasetool_for_byexp and ($req_type eq 'byexp'))) {
             $use_releasetool = 1;
             if ($release_name) {
@@ -599 +624 @@
         $command .= " -data_group $data_group";
     } elsif (!$use_releasetool and $req_type eq 'byskycell') {
-        # XXX: Why am I using default data_groups only for byskycell requests?
+        # Default data_groups are used by bycoord in the camera run lookup and
+        # here for byskycell.
         $command .= $default_data_groups;
     }
@@ -773 +799 @@
         my ($release_name, $survey, $default_tess_id) = get_release_info($row);
         if (!$tess_id and $default_tess_id) {
-            $tess_id = $default_tess_id;
+            if (ref($default_tess_id) eq 'ARRAY') {
+                # XXX: handle the case where default_tess_id is an array
+                # For now just take the first entry (RINGS.V3)
+                $tess_id = $default_tess_id->[0];
+            } else {
+                $tess_id = $default_tess_id;
+            }
         }
 
@@ -1061 +1093 @@
     
 
+    my @tess_ids;
     my ($release_name, $survey, $default_tess_id) = get_release_info($row);
-    if (!$tess_id and $default_tess_id) {
-        $tess_id = $default_tess_id;
+    if ($tess_id) {
+        push @tess_ids, $tess_id;
+    } elsif ($default_tess_id) {
+        if (ref($default_tess_id) eq 'ARRAY') {
+            @tess_ids = @$default_tess_id;
+        } else {
+            push @tess_ids, $default_tess_id;
+        }
     }
 
     my $rowList = [$row];
 
-    my $results = ();
+    my $results = [];
     if (($stage eq "raw") or ($stage eq "chip")) {
 
@@ -1081 +1120 @@
                 next if $component and ($chip->{component} ne $component);
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id},
-                    $tess_id, $chip->{component}, $need_magic, 
+                    # lookup doesn't need tess_id in this context
+                    # $tess_id,
+                    undef, 
+                    $chip->{component}, $need_magic, 
                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
                     $fwhm_min, $fwhm_max,
@@ -1097 +1139 @@
                                               and ($stage ne "stack") and ($stage ne "diff");
 
-        my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
-
-        if (!$skycells or scalar @$skycells == 0) {
+        foreach my $tess_id (@tess_ids) {
+            my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
+
+            if ($skycells and scalar @$skycells != 0) {
+                # XXX: We are not applying the fwhm cuts unless we use releasetool
+                foreach my $skycell (@$skycells) {
+                    my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
+                        $skycell->{tess_id}, $skycell->{component}, $need_magic, 
+                        $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+                        $fwhm_min, $fwhm_max,
+                        undef, 0, $release_name, $survey,
+                        $verbose);
+
+                    next if !$these_results;
+                    push @$results, @$these_results;
+                }
+            }
+        }
+        if (scalar @$results == 0) {
             setErrorCodes($rowList, $PSTAMP_NO_IMAGE_MATCH);
-        } else {
-            # XXX: We are not applying the fwhm cuts unless we use releasetool
-            foreach my $skycell (@$skycells) {
-                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
-                    $skycell->{tess_id}, $skycell->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
-                    $fwhm_min, $fwhm_max,
-                    undef, 0, $release_name, $survey,
-                    $verbose);
-
-                next if !$these_results;
-                push @$results, @$these_results;
-            }
         }
     }
@@ -1976 +2021 @@
 }
 
+# get user supplied release paramters and the asssociated default_tess_id.
+# If release parameters are not supplied set defaults.
 sub get_release_info {
     my $row = shift;
@@ -1985 +2032 @@
         # no release check for survey
         $survey = isnull($row->{SURVEY_NAME}) ? "" : $row->{SURVEY_NAME};
-    } else {
-        # if release is given leave survey empty
-        $survey = "";
-    }
-
-    my $tess_id;
-    if ($release_name) {
-        # hack to optimize bycoord and byskycell lookups for well known releases
-        # XXX: todo query database and return a list for the survey or release
-        if ($release_name eq '3PI.PV1') {
-            $tess_id = 'RINGS.V3';
-        }
-    }
-
-    return ($release_name, $survey, $tess_id);
+    }
+
+    # XXX: From here down this function is currently very PS1 specific and depends on 
+    # conventions being used for release names.
+    # The comments following the lines  marked 'XXX to improve' give suggestions on
+    # how these hacks can be cleaned up. 
+
+    my $tess_id = $row->{TESS_ID};
+    if (!$release_name and !$survey) { 
+        # no release or survey. If user does not supply
+        # a data group use a default survey
+        my $req_type = $row->{REQ_TYPE};
+        if ($req_type eq "bycoord" or $req_type eq 'byskycell') {
+            # USING 3PI for bycoord and byskycell requests
+            # XXX to improve
+            # get default survey from project
+            my $data_group = $row->{DATA_GROUP};
+            if (isnull($data_group) and isnull($tess_id)) {
+                print "No release information supplied for this row. Defaulting to 3PI survey\n";
+                $survey = '3PI';
+            }
+        }
+    }
+
+    # if survey was not supplied, guess it from the release_name. This is used below
+    # to choose the default_tess_id
+    if (!$survey) {
+        if ($release_name) {
+            # XXX: to improve 
+            # get survey from ippRelease(release_name) in database. Would need to cache
+            # such results
+            if ($release_name =~ "^3PI") {
+                $survey = '3PI';
+            } elsif ($release_name =~ "^MD") {
+                $survey = substr($release_name, 0, 4);
+            } elsif ($release_name =~ "^SSS") {
+                $survey = 'SSS';
+            } elsif ($release_name =~ "^SAS") {
+                $survey = 'SAS';
+            } else {
+                # we don't know
+                print "Can't guess survey for $release_name\n";
+            }
+        } elsif (!isnull($tess_id)) {
+            # finally if it is a MD tessellation use the MD field as the survey
+            # (This will also work for M31 and STS when that data is released.
+            if ($tess_id =~ "^MD") {
+                $survey = substr($tess_id, 0, 4);
+            }
+        }
+    }
+
+    # choose default tess_id based on the survey
+
+    # Note: Setting a default_tess_id effectively disables lookup bycoord requests
+    # for previous tessellations unless the user supplies the tess_id (which is
+    # applied by the caller to this function).
+    # bills has decided that this is desirable behaviour since most everything has
+    # been processed on a V3 tess.
+    my $default_tess_id;
+    if ($survey) {
+        # XXX: to improve
+        # Put default tess_id in survey table.
+        # note users will need to supply tessellation to get at 3pi skycells
+        # in the cnp tessellation. Hmm. Maybe we need to be clever and look
+        # at declination or perhaps allow default_tess_id to be an array.
+        if ($survey eq '3PI') {
+            $default_tess_id = ['RINGS.V3', 'CNP.V3'];
+        } elsif ($survey eq 'SSS' or $survey eq 'SAS') {
+            $default_tess_id = 'RINGS.V3';
+        } elsif ($survey =~ 'MD') {
+            $default_tess_id = $survey . '.V3';
+        }
+    }
+            
+    # leave the returned survey blank if release_name was supplied
+    # XXX: Why? Answer: this is how this function always behaved and I'm not sure if the
+    # callers depend on that behavior.
+    $survey = "" if $release_name;
+
+    return ($release_name, $survey, $default_tess_id);
 }
 
@@ -2007 +2120 @@
     if (!isnull($run_type)) {
         $run_type = lc($run_type);
+        # XXX: these diff_mode values should be defined somewhere else.
         if ($run_type eq 'warp_warp') {
             $diff_mode = 1;

tags/ipp-20150329/dbconfig/changes.txt

@@ -2526 +2526 @@
 --
 
-UPDATE dbversion set schema_version = '1.1.80, updated= CURRENT_TIMESTAMP();
+UPDATE dbversion set schema_version = '1.1.80', updated= CURRENT_TIMESTAMP();
 ALTER TABLE remoteComponent ADD COLUMN jobs INT after stage_id;
 ALTER TABLE remoteComponent ADD COLUMN state VARCHAR(64) NOT NULL after jobs;

tags/ipp-20150329/ppSkycell/src/ppSkycellLoop.c

@@ -240 +240 @@
               maxCRPIX1 = wcs->crpix1;
             }
-            if (wcs->crpix2 < maxCRPIX2) {
+            if (wcs->crpix2 > maxCRPIX2) {
               maxCRPIX2 = wcs->crpix2;
             }

tags/ipp-20150329/pstamp/scripts/pstamp_job_run.pl

@@ -92 +92 @@
 my $stack_bkg_mk_mdc = can_run('stack_bkg_mk_mdc.pl') or (warn "Can't find stack_bkg_mk_mdc.pl" and $missing_tools = 1);
 my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1);
-
+my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -177 +177 @@
                 # be copied from the cmf file.
                 $argString .= " -recipe PSPHOT STACKPHOT"
+            }
+        } elsif ($stage eq 'stack') {
+            # no astrom file for stack (skycal cmf) Go find a staticsky cmf and use that
+            # otherwise silently ignore the request for sources since the stack smf is not useful.
+            my $staticsky_cmf = findStaticskyCMF($params);
+            if ($staticsky_cmf) {
+                print "Using staticsky cmf file $staticsky_cmf for sources\n";
+                $argString .= " -write_cmf";
+                $fileArgs  .= " -sources $staticsky_cmf";
+                # Set psphot recipe to STACKPHOT so that the extended source paramters will
+                # be copied from the cmf file.
+                $argString .= " -recipe PSPHOT STACKPHOT";
+                push @file_list, $staticsky_cmf;
+            } else {
+                print "No skycal or staticsky cmf for $params->{stack_id} ignoring request for sources\n";
             }
         } elsif ($params->{cmf}) {
@@ -864 +879 @@
     return 0;
 }
+sub findStaticskyCMF {
+    my $params = shift;
+    my $stack_id = $params->{stack_id};
+
+    &my_die( "NO stack_id found in findStaticskyCMF\n",
+            $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$stack_id;
+
+    my $command = "$staticskytool -dbname $params->{imagedb} -result -stack_id $stack_id";
+    my $results = runToolAndParse($command, $verbose);
+
+    if (!$results) {
+        print "no staticsky results found for stack $stack_id\n";
+        return undef;
+    }
+
+    my $latest;
+    my $latest_sky_id = 0;
+    foreach my $result (@$results) {
+        if ($result->{sky_id} > $latest_sky_id) {
+            $latest = $result;
+            $latest_sky_id = $result->{sky_id};
+        }
+    }
+    if ($latest) {
+        # XXXX Use proper file rule
+        return $latest->{path_base} .  ".stk.$stack_id.cmf";
+    } 
+    # Can't happen can it?
+    return undef;
+}
 
 sub my_die

tags/ipp-20150329/tools/checkexp

@@ -53 +53 @@
         summitExp.fault AS summit_fault, 
         pzDownloadExp.state AS download_state, count(pzDownloadImfile.class_id) AS download_count,
-        newExp.state AS newExp_state, newExp.exp_id
+        newExp.state AS newExp_state, newExp.exp_id,
+        rawExp.state as rawExp_state
         FROM summitExp LEFT JOIN pzDownloadExp USING(exp_name) 
             LEFT JOIN pzDownloadImfile USING(exp_name) 
@@ -88 +89 @@
 my $first_exp_id;
 my $first_exp_name;
+my $last_registered_exp_id = "";
+my $last_registered_exp_name = "";
 while (my $se_ref = $se_stmt->fetchrow_hashref()) {
     push @summit_exps, $se_ref;
@@ -98 +101 @@
     my $imfiles = $se_ref->{imfiles};
     my $dateobs = $se_ref->{dateobs};
+    my $raw_state = $se_ref->{rawExp_state};
     if ($summit_fault) {
         push @summit_faults, $se_ref;
@@ -115 +119 @@
             $last_new_exp_id = $exp_id;
             $last_new_exp_name = $exp_name;
+        }
+        if ($raw_state and $raw_state eq 'full') {
+            $last_registered_exp_id = $exp_id;
+            $last_registered_exp_name = $exp_name;
         }
     }
@@ -180 +188 @@
 
 print "first exposure:  $first_exp_name $first_exp_id\n";
+print "last registered: $last_registered_exp_name $last_registered_exp_id\n" 
+            if $last_registered_exp_id ne $last_exp_id;
 print "last copied:     $last_new_exp_name $last_new_exp_id\n"
             if $last_new_exp_id ne $last_exp_id;

tags/ipp-20150329/tools/mysql-dump/gpc1_install.sh

@@ -13 +13 @@
 #    - Moves the MD5 checksum to the distribution directory
 # 4) Import gpc1 into gpc1_0 or gpc1_1 (ensured by gpc1Import.py utility)
+# NOTE step 4 is no longer done
 . /home/panstarrs/ipp/mysql-dump/functions.sh
 
@@ -60 +61 @@
 fi
 
-log INFO "Symlinking backup file [$TARGET/$BACKUP_FILE] to distribution [$DISTRIBUTION_TARGET]"
+log INFO "Hard linking backup file [$TARGET/$BACKUP_FILE] to distribution [$DISTRIBUTION_TARGET]"
 /usr/bin/ln -f $TARGET/$BACKUP_FILE $DISTRIBUTION_TARGET
 /usr/bin/cp -f $TARGET/$MD5FILE $DISTRIBUTION_MD5
 
+# We no longer ingest the backup into the database. 
 # if it is between 0 and 4, ingest into gpc1_0 or gpc1_1
 #HOUR=`date +"%H"`