Changeset 38052 for branches/eam_branches/ipp-20150112/pstamp/scripts/pstampparse.pl

Timestamp:

Mar 28, 2015, 8:28:32 PM (11 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20150112/pstamp/scripts/pstampparse.pl (modified) (11 diffs)

branches/eam_branches/ipp-20150112/pstamp/scripts/pstampparse.pl

@@ -18 +18 @@
 use POSIX;
 use Time::HiRes qw(gettimeofday);
+use Time::Local;
 
 my $verbose;
@@ -117 +118 @@
 my_die("REQ_NAME not found in $request_file_name\n", $PS_EXIT_PROG_ERROR)  if (!$req_name);
 my_die("wrong EXTVER $extver found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
+
+if ($extver eq "1") {
+    print STDERR "WARNING WARNING WARNING\n";
+    print STDERR "Very soon version 1 postage stamp requests will be rejected. Please update your request tables to version 2 format.\n";
+    print STDERR "Note that EMAIL will not be optional\n";
+
+} elsif (!$email) {
+    my_die("ERROR: Required parameter EMAIL not found in request file header.\n", $PSTAMP_INVALID_REQUEST) 
+}
+    
 
 if ($extver >= 2) {
@@ -260 +271 @@
 my $num_jobs = 0;
 
+my $accessLevel = getUsersAccessLevel($email, $label);
+if ($accessLevel < 0) {
+    print STDERR "Data access forbidden.\n";
+    insertFakeJobForRow(undef, 0, $PSTAMP_NOT_AUTHORIZED);
+    exit 0;
+}
+
 foreach my $row (@$rows) {
 
@@ -271 +289 @@
     $row->{job_num} = 0;
     $row->{error_code} = 0;
+    $row->{accessLevel} = $accessLevel;
 
     $num_jobs += processRow($action, $row);
@@ -537 +556 @@
 }
 
-sub processRows {
-    my $action = shift;
-    my $rowList = shift;
-    my $num_jobs = 0;
-
-    if ($action eq 'LIST') {
-        # LIST is not allowed by caller. Can't get here.
-        return list_targets($rowList);
-    }
-
-    # all rows in the list are compatible
-    my $row = $rowList->[0];
-
-    my $project  = $row->{PROJECT};
-
-    # note: resolve_project avoids running pstamptool every time by remembering the
-    # last project resolved
-    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
-    if (!$proj_hash) {
-        foreach my $r (@$rowList) {
-            insertFakeJobForRow($r, 1, $PSTAMP_UNKNOWN_PRODUCT);
-            $num_jobs++;
-        }
-        return $num_jobs;
-    }
-    my $req_type  = $row->{REQ_TYPE};
-    my $stage     = $row->{IMG_TYPE};
-    my $id        = $row->{ID};
-    my $component = $row->{COMPONENT};
-    my $tess_id   = $row->{TESS_ID};
-
-    my $filter    = $row->{REQFILT};
-    my $mjd_min   = $row->{MJD_MIN};
-    my $mjd_max   = $row->{MJD_MAX};
-    my $data_group = $row->{DATA_GROUP};
-
-    my $rownum     = $row->{ROWNUM};
-    my $job_type   = $row->{JOB_TYPE};
-    my $option_mask= $row->{OPTION_MASK};
-    
-    my $image_db   = $proj_hash->{dbname};
-    my $camera     = $proj_hash->{camera};
-    my $need_magic = $proj_hash->{need_magic};
-    # Since user can get unmagicked data "by coordinate" requests can go back in time
-    # to dredge unusable data from the "dark days"...
-    if ($req_type eq 'bycoord' and $mjd_min eq 0) {
-            # ... so unless the user sets mjd_min clamp it to 2009-04-01
-            # XXX: This value should live in the pstampProject table not be hardcoded here
-            $mjd_min = 54922;
-    }
-
-    $need_magic = 0 if $stage eq 'stack';
-
-    # XXX: magic is dead
-    $need_magic = 0;
-
-    my $numRows = scalar @$rowList;
-
-    if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) {
-        # avoid error from print below if $id isn't needed
-        $id = "" if !$id;
-    }
-    print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n";
-    
-    # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
-    # request specification. An array reference is returned.
-    my $start_locate = gettimeofday();
-
-    # XXX: perhaps we should get rid of most of this argument list.
-    # Now that we are passing down compatible rows all of the
-    # information required is contained there
-
-    my $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
-                $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
-
-    my $dtime_locate = gettimeofday() - $start_locate;
-    print "Time to locate_images for row $rownum $dtime_locate\n";
-
-    # handle this 
-    # XXX: what did I mean by that comment?
-    $row->{need_magic} = $need_magic;
-
-    $num_jobs += queueJobs($action, $rowList, $imageList);
-
-    # if a row slipped through with no jobs add a faulted one
-    foreach my $row (@$rowList) {
-        if ($row->{job_num} == 0) {
-            print "row $row->{ROWNUM} produced no jobs\n";
-            print STDERR "row $row->{ROWNUM} produced no jobs\n";
-            my $error_code = $row->{error_code};
-            $error_code =  $PSTAMP_NO_IMAGE_MATCH if !$error_code;
-            insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
-        }
-    }
-
-    return $num_jobs;
-}
-
 sub queueJobForImage
 {
@@ -648 +569 @@
 
     $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_updates_allowed;
+
+    # determine if there is a cut off date for data access for this user
+    my $mjd_max;
+    my $adjustedDateCuts = adjustDateCuts($row, \$mjd_max);
 
     my $roi_string;
@@ -692 +617 @@
 
     my $imagefile = $image->{image};
-    if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
-        # XXX: should we add a faulted job so the client can know what happened if no images come back?
-        # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback
-        # to users, but speeds up processing significantly
-        print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
-
-        # for now assume yes.
-
-        insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
-        return 1;
-    } elsif ($stage eq "stack") {
+
+    if ($stage eq "stack") {
         # unconvolved stack images weren't available prior to some point in time.
         # XXX: handle this more correctly by examining the stack run's config dump file.
@@ -713 +629 @@
             insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AVAILABLE);
             return 1;
+        }
+    } else { 
+        if ($need_magic and !$image->{magicked}) {
+            # XXX: should we add a faulted job so the client can know what happened if no images come back?
+            # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback
+            # to users, but speeds up processing significantly
+            print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
+
+            # for now assume yes.
+
+            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+            return 1;
+        }
+
+        if ($adjustedDateCuts) {
+            my $mjd = 0;
+            my $exp_id;
+            if ($stage eq 'diff') {
+                # diff stage doesn't have a dateobs so we need to hack it.
+                # Extract dateobs from the exp_name_1
+                if ($row->{PROJECT} eq 'gpc1') {
+                    my $exp_name = $image->{exp_name_1};
+                    if ($exp_name ne 'NULL') {
+                        $mjd = 50000 + (substr($exp_name, 1, 4));
+                        $exp_id = $image->{exp_id_1};
+                    } else {
+                        # it's stack-stack diff just let it go with mjd = 0
+                        # XXX: deal with this
+                    }
+                }
+            } else {
+                $exp_id = $image->{exp_id};
+                $mjd = dateobsToMJD($image->{dateobs});
+            }
+
+            if ($mjd > $mjd_max) {
+                print STDERR "User is not authorized for stamps from this exposure.\n";
+                insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AUTHORIZED, $exp_id);
+                return 1;
+            }
         }
     }
@@ -943 +899 @@
     my $job_num = shift;
     my $fault = shift;
+    my $exp_id = shift;
 
     my ($job_type, $rownum);
@@ -965 +922 @@
     my $command = "$pstamptool -addjob  -req_id $req_id -job_type $job_type"
                         . " -rownum $rownum -state stop -fault $fault";
+    $command .= " -exp_id $exp_id" if $exp_id and $exp_id ne 'NULL';
 
     if (!$no_update) {
@@ -1220 +1178 @@
 }
 
+# Prototype function for setting access level.
+# XXX: Implement something more flexible using the database.
+sub getUsersAccessLevel {
+    my $email = shift;
+    my $label = shift;
+
+    my $level = -1;
+
+    if (!$email) {
+        # No email provided. Eventually this will be forbidden.
+        # For now in order to temporarily continue to support the version 1 request format
+        # set the access rights based on the assigned label.
+        if ($label eq 'IFA' or $label eq 'QUB' or $label =~ 'MOPS') {
+            $level = 0;
+        } else {
+            # access rights PS1 data only
+            $level = 1;
+        }
+    } else {
+        # we've got an email check the domain
+        my ($user, $domain) = split '@', $email;
+        if ($domain) {
+            $domain = lc($domain);
+            if ($domain eq 'ifa.hawaii.edu' or $domain eq 'qub.ac.uk') {
+                $level = 0;
+            } else {
+                # XXX check for special users
+                # access rights PS1 data only
+                $level = 1;
+            }
+        } else {
+            print STDERR "Error: $email is not a valid email address\n";
+        }
+    }
+
+    return $level;
+}
+
+
+# determine the latest exposure date that a user may have access to
+# based on the user's accessLevel.
+sub adjustDateCuts {
+    my $row = shift;
+    my $r_mjd_max = shift;
+    my $adjusted = 0;
+
+    my $MJD_PS1_BEGIN = 54922;    # 2009-04-01
+    my $MJD_PS1_END   = 56838;    # 2014-06-30      XXX get the actual date
+
+    if ($row->{accessLevel} == 1) {
+        # User has access to PS1 data only
+        my $mjd_max_user = $row->{MJD_MAX};
+        if (!$mjd_max_user or $mjd_max_user > $MJD_PS1_END) {
+            $$r_mjd_max = $MJD_PS1_END;
+            $adjusted = 1;
+        }
+    }
+    return $adjusted;
+}
+
+sub dateobsToMJD {
+    my $dateobs = shift;
+
+    my ($date, $time) = split "T", $dateobs;
+
+    my ($year, $mon, $day) = split "-", $date;
+
+    my ($hr, $min, $sec) = split ":",   $time;
+    my $fraction = $sec - int($sec);
+    $sec = int($sec);
+
+    my $ticks = timegm($sec, $min, $hr, $day, $mon-1, $year-1900);
+    $ticks += $fraction;
+
+    # I'm not allowing for leap seconds here but for our purposes here that is ok
+    my $mjd = 40587.0 + ($ticks / 86400.);
+    return $mjd;
+}
+
 sub my_die
 {