Changeset 25744

Timestamp:

Oct 2, 2009, 12:03:23 PM (17 years ago)

Author:

eugene

Message:

merge from head

Location:

branches/eam_branches/20090715

Files:

• DataStore/lib/DataStore/Utils.pm (modified) (1 diff)
• DataStore/scripts/dsget (modified) (1 diff)
• DataStore/scripts/dsproductls (modified) (1 diff)
• DataStoreServer/scripts/dsprodtool (modified) (11 diffs)
• DataStoreServer/scripts/dsreg (modified) (1 diff)
• DataStoreServer/scripts/installscripts (modified) (1 diff)
• DataStoreServer/web/cgi/index.txt (deleted)
• DataStoreServer/web/cgi/index.txt.template (copied) (copied from trunk/DataStoreServer/web/cgi/index.txt.template )
• Ohana/src/getstar/src/dvoImagesAtCoords.c (modified) (3 diffs)
• Ohana/src/relphot/src/StarOps.c (modified) (3 diffs)
• Ohana/src/relphot/src/relphot_objects.c (modified) (1 diff)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (10 diffs)
• archive/ducttape (modified) (1 prop)
• catalyst (modified) (1 prop)
• catalyst/2007.0-specs (modified) (1 prop)
• catalyst/2008.0-specs (modified) (1 prop)
• catalyst/2008.0-specs/hardened (modified) (1 prop)
• dbconfig/changes.txt (modified) (1 diff)
• dbconfig/pstamp.md (modified) (1 diff)
• doc/misc/docgen.pl (modified) (1 prop)
• extperl/Params-Validate-0.92.tar.gz (copied) (copied from trunk/extperl/Params-Validate-0.92.tar.gz )
• hardware (modified) (1 prop)
• ippScripts/scripts/dist_component.pl (modified) (1 diff)
• ippScripts/scripts/dist_make_fileset.pl (modified) (6 diffs)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (1 diff, 1 prop)
• ippTasks/addstar.pro (modified) (2 diffs)
• ippTasks/camera.pro (modified) (2 diffs)
• ippTasks/chip.pro (modified) (2 diffs)
• ippTasks/destreak.pro (modified) (2 diffs)
• ippTasks/fake.pro (modified) (2 diffs)
• ippTasks/pantasks.pro (modified) (2 diffs)
• ippTasks/rcserver.pro (modified) (2 diffs)
• ippTasks/warp.pro (modified) (2 diffs)
• ippTests/tap (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/disttool_pendingfileset.sql (modified) (1 diff)
• ippTools/share/disttool_revertcomponent.sql (modified) (1 prop)
• ippTools/share/disttool_revertrun.sql (modified) (1 prop)
• ippTools/share/pstamptool_listjob.sql (copied) (copied from trunk/ippTools/share/pstamptool_listjob.sql )
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff)
• ippTools/src/chiptool.c (modified) (2 diffs)
• ippTools/src/chiptoolConfig.c (modified) (1 diff)
• ippTools/src/difftool.c (modified) (2 diffs)
• ippTools/src/difftoolConfig.c (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (3 diffs)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff)
• ippTools/src/regtool.c (modified) (2 diffs)
• ippTools/src/regtoolConfig.c (modified) (1 diff)
• ippTools/src/stacktool.c (modified) (2 diffs)
• ippTools/src/stacktoolConfig.c (modified) (1 diff)
• ippTools/src/warptool.c (modified) (2 diffs)
• ippTools/src/warptoolConfig.c (modified) (1 diff)
• ppStack/src/ppStackLoop.c (modified) (6 diffs)
• psconfig/tagsets/ipp-2.9.perl (modified) (2 diffs)
• pstamp/scripts/Makefile.am (modified) (1 diff)
• pstamp/scripts/pstamp_finish.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_request_file (copied) (copied from trunk/pstamp/scripts/pstamp_request_file )
• pstamp/scripts/pstamp_results_file.pl (modified) (2 diffs)
• pstamp/scripts/pstampparse.pl (modified) (12 diffs)
• pstamp/src/Makefile.am (modified) (1 diff)
• pstamp/src/ppstamp.h (modified) (1 diff)
• pstamp/src/ppstampMakeStamp.c (modified) (9 diffs)
• pstamp/src/pstamp.h (modified) (1 diff)
• pstamp/src/pstampGetROI.c (modified) (1 diff)
• pstamp/src/pstampint.h (copied) (copied from trunk/pstamp/src/pstampint.h )
• pstamp/src/pstamprequest.c (modified) (1 diff)
• pstamp/test/asteroid.txt (modified) (1 diff)
• pstamp/test/byskycell.txt (copied) (copied from trunk/pstamp/test/byskycell.txt )
• pstamp/test/gpc1_sample.txt (modified) (1 diff)
• pstamp/test/pstamp_req_create (modified) (1 diff)
• pswarp/src/pswarpDefine.c (modified) (1 diff)
• pswarp/src/pswarpLoop.c (modified) (1 diff)

branches/eam_branches/20090715/DataStore/lib/DataStore/Utils.pm

@@ -42 +42 @@
 $BYTE_FIELD = qr/^\d+$/;
 $MD5_FIELD = qr/^[0-9a-f]{32}$/;
-%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz);
-%KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS);
-%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table );
+%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits );
+%KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS IPP-DIST);
+%KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc);
 
 =pod

branches/eam_branches/20090715/DataStore/scripts/dsget

@@ -207 +207 @@
     if (defined $copies and $copies > 1) {
         foreach (1 .. ($copies - 1)) {
-            $neb->replicate($filename, 'any')
+            $neb->replicate($filename)
                 or die "failed to replicate $filename failed: $!";
         }

branches/eam_branches/20090715/DataStore/scripts/dsproductls

@@ -59 +59 @@
 
 print "# uri fileset datetime type\n";
-my $num = 0;
 foreach my $fs (@$data) {
-    $num++;
-    if (!defined $fs->uri) {
-        # Somewhow we get here occasaionally I don't know why yet
-        print STDERR "uri is not defined! skipping fs: $num\n";
-        next;
-    }
     print $fs->uri, " ", $fs->fileset, " ", $fs->datetime, " ", $fs->type;
     if ($extra) {

branches/eam_branches/20090715/DataStoreServer/scripts/dsprodtool

@@ -12 +12 @@
 use PS::IPP::Metadata::Config;
 use File::Copy;
+
+use Term::ReadKey;
 
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
@@ -36 +38 @@
 my $dsroot;
 my $dbname;
+my $dbpass;
 
 my $add;
@@ -78 +81 @@
 } else {
     $product = $del;
+    if (! -t STDIN) {
+        die "cannot delete product from script\n";
+    }
+    print "*** Delete the Data Store Product $product? ***\n";
+    print "*** to delete $product answer YES, and give password ***\n";
+    print "*** WARNING this action is permanant *** \n";
+    print "Delete? (YES/[n]): ";
+
+    my $line = ReadLine(0);
+    chomp $line;
+    exit 1 if !$line or ($line ne "YES");
+
+    print "password: ";
+    ReadMode('noecho');
+    $line = ReadLine(0);
+    ReadMode('normal');
+    print "\n";
+    chomp $line;
+    $dbpass = $line;
 }
 
@@ -93 +115 @@
 }
 
-my $root_index_script = "$dsroot/index.txt";
+my $root_index_script = "$dsroot/index.txt.template";
 if (!stat($root_index_script)) {
     $err .= "Data Store not found at '$dsroot'.\n"
@@ -106 +128 @@
     unless defined $dbname;
 my $dbuser   = metadataLookupStr($siteConfig, 'DS_DBUSER');
-my $dbpass   = metadataLookupStr($siteConfig, 'DS_DBPASSWORD');
+$dbpass   = metadataLookupStr($siteConfig, 'DS_DBPASSWORD') if !$dbpass;
 exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass;
 
 my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
-
-my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";
-
 
 my $product_dir = "$dsroot/$product";
@@ -118 +137 @@
 
 if ($del) {
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";
     #
     # delete product
@@ -129 +149 @@
     }
     my $prod_id = $prod_row->{prod_id};
-
-    # if requested, remove the product directory
-    if ($remove) {
-        if (system "rm -r $product_dir") {
-            die("failed to remove $product_dir");
-        }
-    } else  {
-        # otherwise just zap the index script
-        unlink("$index_script_name");
-    }
 
 
@@ -152 +162 @@
     $dbh->do("DELETE from dsProduct where prod_id = $prod_id");
 
+    print "Product $product deleted.\n";
+
+    # if requested, remove the product directory
+    if ($remove) {
+        print "Removing files from $product.\n";
+        if (system "rm -r $product_dir") {
+            die("failed to remove $product_dir");
+        }
+    } else  {
+        # otherwise just zap the index script
+        unlink("$index_script_name");
+    }
+
     exit 0;
 
@@ -158 +181 @@
     # add a new product
     #
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";
+
     my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = \'$product\'");
     $stmt->execute();
@@ -167 +192 @@
 
     # set up the product directory
+    # if there is an old index file delete it
+    if (-e $index_script_name ) {
+        if (!unlink($index_script_name)) {
+            die("failed trying to remove old $index_script_name");
+        }
+    }
     if (! -e $product_dir) {
         $we_created_dir = 1;
@@ -172 +203 @@
             die("failed trying to make product directory $product_dir");
         }
-    }
-    if (-e $index_script_name ) {
-        if (!unlink($index_script_name)) {
-            die("failed trying to remove old $index_script_name");
-        }
-    }
+    } else {
+        # directory alrady exists make sure that it's empty
+        my @dirlist = glob("$product_dir/*");
+        die ("existing product directory $product_dir is not empty") if scalar @dirlist;
+    }
+
     if (!copy($root_index_script, $index_script_name)) {
         print STDERR "failed trying to copy($root_index_script, $index_script_name)";

branches/eam_branches/20090715/DataStoreServer/scripts/dsreg

@@ -203 +203 @@
                 $new_last_fs = $new_newest->{fileset_name};
             } else {
-                $new_last_fs = "none";
+                $new_last_fs = undef;
             }
             $stmt->finish();
-            $dbh->do("UPDATE dsProduct SET last_fs = \'$new_last_fs\' WHERE prod_id = $prod_id");
+            $dbh->do("UPDATE dsProduct SET last_fs = ? WHERE prod_id = $prod_id", undef, ($new_last_fs));
         }
         $dbh->commit();

branches/eam_branches/20090715/DataStoreServer/scripts/installscripts

@@ -56 +56 @@
 # index.txt file for the root of the data store.
 my @dsroot_list = qw(
-index.txt
+index.txt.template
 );
 

branches/eam_branches/20090715/Ohana/src/getstar/src/dvoImagesAtCoords.c

@@ -1 +1 @@
 # include "dvoImagesAtCoords.h"
+
+// We'd like to do this but since pstamp gets built later this won't work
+// so just redefine the macro
+// # include "pstamp.h"
+#define PSTAMP_NO_OVERLAP 28
 
 static int readPoints(char *filename, Point **pointsOut);
@@ -19 +24 @@
   Npoints = readPoints(argv[1], &points);
   if (!Npoints) {
-    exit(0);
+    exit(1);
   }
 
@@ -49 +54 @@
   if (MatchCoords (dbImages, NdbImages, points, Npoints)) {
     ListImagesAtCoords(dbImages, points, Npoints);
+    exit(0);
+  } else {
+    exit(PSTAMP_NO_OVERLAP);
   }
-  // XXX: should we exit with nonzero status if no matches were found?
-  exit (0);
 }
 

branches/eam_branches/20090715/Ohana/src/relphot/src/StarOps.c

@@ -193 +193 @@
   for (i = 0; i < Ncatalog; i++) {
     for (j = 0; j < catalog[i].Naverage; j++) {
+
+      // update average photometry for each of the average filters
       for (Ns = 0; Ns < Nsecfilt; Ns++) {
 
@@ -207 +209 @@
           if (isnan(Msys)) continue;
 
-          // XXX this is a hack for the 2MASS search; better to save an average value?
-          if (catalog[i].measure[m].psfQual < 0.85) continue;
+          // XXX only apply this filter for psphot data from GPC1 for now...
+          if (0 && (catalog[i].measure[m].photcode > 10000) && (catalog[i].measure[m].photcode < 10500)) {
+              if (catalog[i].measure[m].psfQual < 0.85) continue;
+          }
 
           list[N] = Msys - catalog[i].measure[m].Mcal;
@@ -224 +228 @@
         catalog[i].secfilt[Nsecfilt*j+Ns].Ncode = N;
         catalog[i].secfilt[Nsecfilt*j+Ns].Nused = stats.Nmeas;
+      }
+
+      // update average flags based on the detection stats.  
+      // XXX we need to clean this up -- this is a serious set of hacks...
+      if (0) { 
+        int Galaxy2MASS, GalaxySDSS, goodPS1, nEXT, nPSF, good2MASS;
+
+        Galaxy2MASS = FALSE; // best guess for galaxy based on 2MASS J measurements (gal_contam == measure.flags[0x00400000 | 0x00800000])
+        GalaxySDSS = FALSE;  // best guess for galaxy based on SDSS measurements (XXX need to fix SDSS flags)
+        goodPS1 = FALSE;     // true if any PS1 measurements have psfQual > 0.85
+        good2MASS = FALSE;   // true if 2MASS J measurements have significant detections
+        nEXT = nPSF = 0;     // number of PS1 PSF vs EXT measurements
+
+        m = catalog[i].average[j].measureOffset;
+        for (k = 0; k < catalog[i].average[j].Nmeasure; k++, m++) {
+
+          // PS1 data :
+          if ((catalog[i].measure[m].photcode >= 10000) && (catalog[i].measure[m].photcode <= 10500)) {
+            if (catalog[i].measure[m].psfQual > 0.85) {
+              goodPS1 = TRUE;
+              if (!isnan(catalog[i].measure[m].Map)) {
+                if (catalog[i].measure[m].M - catalog[i].measure[m].Map > 0.5) {
+                  nEXT ++;
+                } else {
+                  nPSF ++;
+                }
+              }
+            }
+          }
+          
+          // 2MASS data:
+          if (catalog[i].measure[m].photcode == 2011) {
+            if (catalog[i].measure[m].photFlags & 0x00c00000) {
+              Galaxy2MASS = TRUE;
+            }
+            if (catalog[i].measure[m].photFlags & 0x00000007) {
+              good2MASS = TRUE; 
+            }
+          }  
+        }
+
+        if (nEXT && (nEXT > nPSF)) {
+          catalog[i].average[j].flags |= 0x00010000;
+        }
+        if (goodPS1) {
+          catalog[i].average[j].flags |= 0x00020000;
+        }
+        if (Galaxy2MASS) {
+          catalog[i].average[j].flags |= 0x00040000;
+        }
+        if (good2MASS) {
+          catalog[i].average[j].flags |= 0x00080000;
+        }
       }
     }

branches/eam_branches/20090715/Ohana/src/relphot/src/relphot_objects.c

@@ -68 +68 @@
     
     if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename);
+    
+    // we can optionally convert output format here
+    // but it would be better to define a dvo crawler program to do this
+    // catalog.catformat = DVO_FORMAT_PS1_V1;  
     dvo_catalog_save (&catalog, VERBOSE); 
     dvo_catalog_unlock (&catalog);

branches/eam_branches/20090715/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -34 +34 @@
     my $img_type = shift;   # required
     my $id       = shift;   # required unless req_type eq bycoord or byskycell
+    my $tess_id  = shift;
     my $component= shift;   # class_id or skycell_id
     my $inverse  = shift;
+    my $need_magic = shift;
     my $x        = shift;
     my $y        = shift;
@@ -41 +43 @@
     my $mjd_max  = shift;
     my $filter   = shift;
+    my $label    = shift;
     my $verbose  = shift;
 
 
     # we die in response to bad data in request files
-    die "Unknown req_type: $req_type" if ($req_type ne "byid") and ($req_type ne "byexp")
-                                        and ($req_type ne "bycoord") and ($req_type ne "bydiff");
+    die "Unknown req_type: $req_type" 
+        if ($req_type ne "byid") and
+           ($req_type ne "byexp") and 
+           ($req_type ne "bycoord") and 
+           ($req_type ne "bydiff") and 
+           ($req_type ne "byskycell");
+
+    my $dateobs_begin;
+    my $dateobs_end;
+    if (!iszero($mjd_min)) {
+        $dateobs_begin = mjd_to_dateobs($mjd_min);
+    }
+    if (!iszero($mjd_max)) {
+        $dateobs_end = mjd_to_dateobs($mjd_max);
+    }
 
     if (($req_type eq "byid") and ($img_type eq "diff")) {
@@ -95 +111 @@
         # regtool -dbname $image_db -processedimfile -time_begin $mjd_min -time_end = $mjd_max -filter $filter
         #
-        my $results = lookup_bycoord($ipprc, $image_db, $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+        my $results = lookup_bycoord($ipprc, $image_db, $x, $y, $dateobs_begin, $dateobs_end, $filter, $verbose);
 
         # now take the results and lookup byexp
@@ -102 +118 @@
         $req_type = "byexp";
         $id = $results->{exp_name};
-    }
-
-    my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $component, $verbose);
+    } elsif ($req_type eq "byskycell") {
+        if (($img_type eq "raw") or ($img_type eq "chip")) {
+            print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n";
+            return undef;
+        }
+        if (!$tess_id or !$component) {
+            print STDERR "component and tess_id are required for REQ_TYPE byskycell\n";
+            return undef;
+        }
+    }
+
+    my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic, $dateobs_begin, $dateobs_end, $filter, $label, $verbose);
 
     return $results;
@@ -115 +140 @@
     my $img_type = shift;
     my $id       = shift;
+    my $tess_id  = shift;
     my $component= shift;
+    my $need_magic = shift;
+    my $dateobs_begin = shift;
+    my $dateobs_end   = shift;
+    my $filter = shift;
+    my $label = shift;
     my $verbose  = shift;
 
@@ -139 +170 @@
     my $skycell_id;
 
-    # special class_id value "null" means ignore
-    if ($component and ($component eq "null")) {
+    if (isnull($component)) {
         $component = undef;
     }
 
+    my $magic_arg = $need_magic ? " -destreaked" : "";
     if ($img_type eq "raw") {
         $class_id = $component;
@@ -196 +227 @@
     } elsif ($req_type eq "byexp") {
         $command .= " -exp_name $id";
+    } elsif ($req_type eq "byskycell") {
+        $command .= " -tess_id $tess_id -skycell_id $skycell_id";
     } else {
         die "Unknown req_type supplied: $req_type";
     }
+
+    if ($img_type ne "stack") {
+        $command .= $magic_arg;
+        $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
+        $command .= " -dateobs_end $dateobs_end" if $dateobs_end;
+    }
+
+    $command .= " -filter $filter" if !isnull($filter);
+    $command .= " -label $label" if !isnull($label);
 
     # run the tool and parse the output
@@ -386 +428 @@
     my $x        = shift;
     my $y        = shift;
-    my $mjd_min  = shift;
-    my $mjd_max  = shift;
+    my $dateobs_begin  = shift;
+    my $dateobs_end  = shift;
     my $filter   = shift;
     my $verbose  = shift;
@@ -403 +445 @@
 
     my $args;
-    if ($mjd_min) {
-        my $dateobs_min = mjd_to_dateobs($mjd_min);
-        $args .= " -dateobs_begin $dateobs_min";
-    }
-    if ($mjd_max) {
-        my $dateobs_max = mjd_to_dateobs($mjd_max);
-        $args .= " -dateobs_end $dateobs_max";
-    }
-    if ($filter) {
+    if (!isnull($dateobs_begin)) {
+        $args .= " -dateobs_begin $dateobs_begin";
+    }
+    if (!isnull($dateobs_end)) {
+        $args .= " -dateobs_end $dateobs_end";
+    }
+    if (!isnull($filter)) {
         $args .= " -filter $filter";
     }
@@ -567 +607 @@
     my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks);
 
-    return sprintf "'%4d-%02d-%02d %02d:%02d:%02d'", $year+1900, $mon+1, $day, $hr, $min, $sec;
+    return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec;
+}
+
+sub isnull {
+    my $val = shift;
+
+    return (!defined($val) or (lc($val) eq "null"));
+}
+
+sub iszero {
+    my $val = shift;
+    return (!defined($val) or ($val == 0));
 }
 

branches/eam_branches/20090715/dbconfig/changes.txt

@@ -1298 +1298 @@
 
 ALTER TABLE distRun ADD COLUMN magic_ds_id BIGINT AFTER stage_id;
+
+ALTER TABLE pstampJob ADD COLUMN options BIGINT;

branches/eam_branches/20090715/dbconfig/pstamp.md

@@ -39 +39 @@
     exp_id      S64         0
     outputBase  STR         255
+    options     S64         64
 END

branches/eam_branches/20090715/ippScripts/scripts/dist_component.pl

@@ -164 +164 @@
             or ($file_rule =~ /.JPEG2/));
 
+    if ($stage eq "diff") {
+        next if $file_rule =~ /CONV/;
+    }
+
     my $file_name = $file->{name};
     my $base = basename($file_name);

branches/eam_branches/20090715/ippScripts/scripts/dist_make_fileset.pl

@@ -40 +40 @@
 # Parse the command-line arguments
 my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $dest_id, $product_name, $ds_dbhost, $ds_dbname);
+my ($label, $filter);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -50 +51 @@
            'dest_id=s'      => \$dest_id,    # id for the product
            'product_name=s' => \$product_name,  # location of the data store directory for this product
+           'label=s'       => \$label,
+           'filter=s'      => \$filter,
            'ds_dbhost=s'    => \$ds_dbhost,  # database host for the datastore database
            'ds_dbname=s'    => \$ds_dbname,  # database name for the datastore database
@@ -60 +63 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --dest_id --ds_dbhost --ds_dbname",
+pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --label --filter --dest_id --ds_dbhost --ds_dbname",
            -exitval => 3) unless
     defined $dist_id and
@@ -67 +70 @@
     defined $stage and
     defined $stage_id and
+    defined $label and
+    defined $filter and
     defined $dest_id and
     defined $product_name and
@@ -147 +152 @@
 
 {
-    # XXX: need to chose an appropriate file set type
-    my $command = "$dsreg --add $fileset_name --product $product_name --type notset --list $listFileName";
+    my $command = "$dsreg --add $fileset_name --product $product_name --type IPP-DIST --list $listFileName";
 
     # the data store will refer to the distribution bundle via symlinks back to distRun.outdir
@@ -157 +161 @@
 
     $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $prod_col_3";
-
-    $command .= " --dbname $ds_dbname";    # XXX: notyet --dbhost $ds_dbhost
+    $command .= " --ps4 $label --ps5 $filter";
+
+    $command .= " --dbname $ds_dbname --dbhost $ds_dbhost";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

branches/eam_branches/20090715/ippScripts/scripts/magic_destreak_revert.pl

@@ -193 +193 @@
 revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
 
-if ($stage eq "diff") {
+if ($stage eq "diff" and $inverse) {
     my $name = "PPSUB.INVERSE";
     $image  = $ipprc->filename($name, $path_base);

branches/eam_branches/20090715/ippTasks/addstar.pro

@@ -23 +23 @@
     active true
   end
+  task addstar.revert
+    active true
+  end
 end
 
@@ -30 +33 @@
   end
   task addstar.exp.run
+    active false
+  end
+  task addstar.revert
     active false
   end

branches/eam_branches/20090715/ippTasks/camera.pro

@@ -23 +23 @@
     active true
   end
+  task camera.revert
+    active true
+  end
 end
 
@@ -30 +33 @@
   end
   task camera.exp.run
+    active false
+  end
+  task camera.revert
     active false
   end

branches/eam_branches/20090715/ippTasks/chip.pro

@@ -26 +26 @@
     active true
   end
+  task chip.revert
+    active true
+  end
 end
 
@@ -36 +39 @@
   end
   task chip.advanceexp
+    active false
+  end
+  task chip.revert
     active false
   end

branches/eam_branches/20090715/ippTasks/destreak.pro

@@ -172 +172 @@
 
   task.exec
+    stdout $LOGSUBDIR/destreak.run.log
+    stderr $LOGSUBDIR/destreak.run.log
     book npages magicToDS -var N
     if ($N == 0) break
@@ -362 +364 @@
 
   task.exec
+    stdout $LOGSUBDIR/destreak.revert.run.log
+    stderr $LOGSUBDIR/destreak.revert.run.log
+
     book npages magicDSToRevert -var N
     if ($N == 0) break

branches/eam_branches/20090715/ippTasks/fake.pro

@@ -26 +26 @@
     active true
   end
+  task fake.revert
+    active true
+  end
 end
 
@@ -36 +39 @@
   end
   task fake.advanceexp
+    active false
+  end
+  task fake.revert
     active false
   end

branches/eam_branches/20090715/ippTasks/pantasks.pro

@@ -1 +1 @@
 ## pantasks.pro : globals and support macros : -*- sh -*-
 
+# globals that may be modified by the user -- only init if not set
+if ($?NEBULOUS == 0)      set NEBULOUS = 0
+if ($?NETWORK == 0)       set NETWORK = 1
+if ($?PARALLEL == 0)      set PARALLEL = 1
+if ($?VERBOSE == 0)       set VERBOSE = 1
+if ($?LABEL:n == 0)       set LABEL:n = 0
+if ($?POLLLIMIT == 0)     set POLLLIMIT = 32
+if ($?KEEP_FAILURES == 0) set KEEP_FAILURES = 0
+
+if ($?LOGDIR == 0) 
+  $LOGDIR = `pwd`
+  $LOGDIR = $LOGDIR/pantasks_logs
+  mkdir $LOGDIR
+end
+
 # globals used to control system behavior : these should be in uppercase
-$NEBULOUS = 0
-$NETWORK = 1
-$PARALLEL = 1
-$VERBOSE = 1
-$LABEL:n = 0
-$POLLLIMIT = 32
-$LOGDIR = `pwd`
-$LOGDIR = $LOGDIR/pantasks_logs
-mkdir $LOGDIR
-$KEEP_FAILURES = 0
-
 $LOADPOLL = 1.0
 $LOADEXEC = 5.0
@@ -144 +148 @@
   for i 0 $LABEL:n
     echo $LABEL:$i
+  end
+end
+
+macro save.labels
+  if ($0 != 1)
+    echo "USAGE: save.labels"
+    break
+  end
+  if ($?LABEL:n == 0)
+    echo "no labels defined"
+  end
+  if ($LABEL:n == 0)
+    echo "no labels defined"
+  end
+
+  exec rm -f labels.list
+  local i
+  for i 0 $LABEL:n
+    exec echo $LABEL:$i >> labels.list
+  end
+end
+
+macro load.labels
+  if ($0 != 1)
+    echo "USAGE: load.labels"
+    break
+  end
+
+  file labels.list found
+  if (not($found)) 
+    echo "no saved labels in labels.list"
+    return
+  end
+ 
+  list mylabels -x "cat labels.list"
+  local i
+  for i 0 $mylabels:n
+    add.label $mylabels:$i
   end
 end

branches/eam_branches/20090715/ippTasks/rcserver.pro

@@ -129 +129 @@
     book getword rcPendingFS $pageName stage -var STAGE
     book getword rcPendingFS $pageName stage_id -var STAGE_ID
+    book getword rcPendingFS $pageName label -var LABEL
+    book getword rcPendingFS $pageName filter -var FILTER
     book getword rcPendingFS $pageName dist_dir -var DIST_DIR
 #    book getword rcPendingFS $pageName clean -var CLEAN
@@ -147 +149 @@
     book setword rcPendingFS $pageName pantaskState RUN
 
-    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --dest_id $DEST_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR
+    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --label $LABEL --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR
 
     add_standard_args run

branches/eam_branches/20090715/ippTasks/warp.pro

@@ -49 +49 @@
     active true
   end
+  task warp.revert.overlap
+    active true
+  end
+  task warp.revert.warped
+    active true
+  end
 end
 
@@ -66 +72 @@
   end
   task warp.advancerun
+    active false
+  end
+  task warp.revert.overlap
+    active false
+  end
+  task warp.revert.warped
     active false
   end

branches/eam_branches/20090715/ippTools/share/Makefile.am

@@ -195 +195 @@
      pstamptool_completedreq.sql \
      pstamptool_datastore.sql \
+     pstamptool_listjob.sql \
      pstamptool_pendingjob.sql \
      pstamptool_pendingreq.sql \

branches/eam_branches/20090715/ippTools/share/disttool_pendingfileset.sql

@@ -4 +4 @@
     distRun.stage,
     stage_id,
+    distTarget.label,
+    distTarget.filter,
     CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir,
     rcDestination.name AS product_name,

branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql

@@ -1249 +1249 @@
         exp_id BIGINT,
         outputBase VARCHAR(255),
+        options BIGINT,
         PRIMARY KEY(job_id, req_id),
         KEY(job_id),

branches/eam_branches/20090715/ippTools/src/chiptool.c

@@ -603 +603 @@
     PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
     pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
-    PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "==");
+    PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "==");
+
+    PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args,     "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false);
 
     if (!psListLength(where->list) &&
@@ -631 +635 @@
         // don't list faulted rows
         psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0");
+    }
+    if (not_destreaked) {
+        if (destreaked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense");
+            return false;
+        }
+        if (magicked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense");
+            return false;
+        }
+        psStringAppend(&query, " AND chipProcessedImfile.magicked = 0");
+    }
+    if (destreaked) {
+        psStringAppend(&query, " AND chipProcessedImfile.magicked != 0");
     }
 

branches/eam_branches/20090715/ippTools/src/chiptoolConfig.c

@@ -158 +158 @@
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-reduction",          0, "search by reduction class", NULL);
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL);
-    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "search by magicked status", false);
+
+    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-destreaked",  0,      "search for destreaked images", false);
+    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-not_destreaked",  0,  "search for images that have not been destreaked", false);
+    psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "search by magicked value", 0);
     psMetadataAddU64(processedimfileArgs,  PS_LIST_TAIL, "-limit",  0,           "limit result set to N items", 0);
     psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all",  0,            "list everything without search terms", false);

branches/eam_branches/20090715/ippTools/src/difftool.c

@@ -665 +665 @@
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
-    PXOPT_COPY_S64(config->args, where,  "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
     PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "==");
-    PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "==");
-
+    PXOPT_COPY_S64(config->args, where,  "-magicked", "diffSkyfile.magicked", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE");
+
+    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
+    if (!template) {
+        PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "==");
+        PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "==");
+        PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "==");
+        PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs",  ">=");
+        PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawInput.dateobs",  "<=");
+        PXOPT_COPY_STR(config->args, where, "-filter",     "rawInput.filter", "==");
+    } else {
+        PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "==");
+        PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "==");
+        PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "==");
+        PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs",  ">=");
+        PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawTemplate.dateobs",  "<=");
+        PXOPT_COPY_STR(config->args, where, "-filter",     "rawTemplate.filter", "==");
+    }
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    PXOPT_LOOKUP_U64(magicked, config->args,        "-magicked", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args,     "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false);
 
     psString query = pxDataGet("difftool_skyfile.sql");
@@ -685 +706 @@
         psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
+    } else if (!all) {
+        psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required");
     }
     psFree(where);
+
+    if (not_destreaked) {
+        if (destreaked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense");
+            return false;
+        }
+        if (magicked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense");
+            return false;
+        }
+        psStringAppend(&query, " AND diffSkyfile.magicked = 0");
+    }
+    if (destreaked) {
+        psStringAppend(&query, " AND diffSkyfile.magicked != 0");
+    }
+
 
     // treat limit == 0 as "no limit"

branches/eam_branches/20090715/ippTools/src/difftoolConfig.c

@@ -123 +123 @@
     // -diffskyfile
     psMetadata *diffskyfileArgs = psMetadataAlloc();
-    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,            "search by diff ID", 0);
-    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id",  0,       "define skycell ID", NULL);
-    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_skyfile_id", 0,    "search by diff_skyfile_id ID", 0);
-    psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id",  0,            "define tessellation ID", NULL);
-    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "define exposure ID", 0);
-    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name",  0,         "define exposure name", NULL);
-    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id",  0,         "define warp_id", NULL);
-    psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
-    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
-    psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "define fault code", 0);
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL,  "-diff_id", 0,           "search by diff ID", 0);
+    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id",  0,      "search by skycell ID", NULL);
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL,  "-diff_skyfile_id", 0,   "search by diff_skyfile_id ID", 0);
+    psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID", NULL);
+    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id",  0,         "search by warp_id", NULL);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-template",  0,        "apply exposure args to template of bothways diff", false);
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL,  "-exp_id",  0,           "search by exposure ID", 0);
+    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name",  0,        "search by exposure name", NULL);
+    psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0,    "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_end", 0,      "search for exposures by time (<=)", NULL);
+    psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL,  "-filter", 0,           "search for filter", NULL);
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL,  "-magicked", 0,         "search by magicked value", 0);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-destreaked",  0,      "search for destreaked images", false);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-not_destreaked",  0,  "search for images that are not destreaked", false);
+    psMetadataAddStr(diffskyfileArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL);
+    psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL,  "-fault",  0,           "search by fault code", 0);
+
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
+    psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
 
     // -revertdiffskyfile

branches/eam_branches/20090715/ippTools/src/pstamptool.c

@@ -544 +544 @@
     PXOPT_LOOKUP_S16(fault,       config->args, "-fault",      false, false);
     PXOPT_LOOKUP_S64(exp_id,      config->args, "-exp_id",     false, false);
+    PXOPT_LOOKUP_S64(options,     config->args, "-options",     false, false);
 
     // unless the job is being inserted with stop state require outputBase
@@ -568 +569 @@
             fault, 
             exp_id, 
-            outputBase
+            outputBase,
+            options
             )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -592 +594 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", false, false);
-    PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false);
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fault",  "fault", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = NULL;
-
-    if (!req_id && !job_id) {
-        fprintf(stderr, "either req_id or job_id must be specified\n");
+    if (!psListLength(where->list)) {
+        fprintf(stderr, "search arguments are required\n");
         exit (1);
     }
 
-    if (req_id) {
-        psStringAppend(&query,
-                "SELECT"
-                " pstampJob.*, pstampRequest.name, pstampRequest.outProduct"
-                " FROM pstampJob"
-                " JOIN pstampRequest USING(req_id)"
-                " WHERE req_id = %" PRId64, req_id
-            );
-    } else {
-        psStringAppend(&query,
-                "SELECT"
-                " pstampJob.*, pstampRequest.name, pstampRequest.outProduct"
-                " FROM pstampJob"
-                " JOIN pstampRequest USING(req_id)"
-                " WHERE job_id = %" PRId64, job_id
-            );
-    } 
+    psString query = pxDataGet("pstamptool_listjob.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // use psDBGenerateWhereSQL because the SQL yields an intermediate table
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob");
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
 
     // treat limit == 0 as "no limit"

branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c

@@ -105 +105 @@
     // -addjob
     psMetadata *addjobArgs = psMetadataAlloc();
-    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0,            "define job req_id", 0); 
-    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0,            "define job rownum", NULL); 
-    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0,            "define job job_type", "stamp"); 
-    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0,            "define job outputBase", NULL); 
-    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0,            "new state", "run"); 
-    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0,           "exposure id", 0); 
-    psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0,            "new result", 0); 
+    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0,           "define job req_id", 0); 
+    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0,           "define job rownum", NULL); 
+    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0,         "define job job_type", "stamp"); 
+    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0,       "define job outputBase", NULL); 
+    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0,            "define state", "run"); 
+    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0,           "define exposure id", 0); 
+    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0,          "define options", 0); 
+    psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0,            "define job result", 0); 
 
     // -listjob
     psMetadata *listjobArgs = psMetadataAlloc();
-    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0,            "define request", 0); 
-    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0,            "define job", 0); 
-    psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
-    psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0,            "use the simple output format", false);
+    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0,          "select by request ID", 0); 
+    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0,          "select by job ID", 0); 
+    psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0,           "select by fault", 0); 
+    psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit",  0,          "limit result set to N items", 0);
+    psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0,         "use the simple output format", false);
 
     // -pendingjob

branches/eam_branches/20090715/ippTools/src/regtool.c

@@ -333 +333 @@
     PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs",  ">=");
     PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "dateobs",  "<=");
+    PXOPT_COPY_STR(config->args, where,  "-filter",        "filter", "==");
+    PXOPT_COPY_S64(config->args, where,  "-magicked",      "magicked", "==");
+
+    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args,     "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -360 +366 @@
         psStringAppend(&query, " %s", "AND rawImfile.fault = 0");
     }
+
+    if (not_destreaked) {
+        if (destreaked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense");
+            return false;
+        }
+        if (magicked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense");
+            return false;
+        }
+        psStringAppend(&query, " AND rawImfile.magicked = 0");
+    }
+    if (destreaked) {
+        psStringAppend(&query, " AND rawImfile.magicked != 0");
+    }
+
 
     // add the ORDER BY statement if desired

branches/eam_branches/20090715/ippTools/src/regtoolConfig.c

@@ -136 +136 @@
     ADD_OPT(Str,  processedimfileArgs, "-exp_name",  "search by exposure name",               NULL);
     ADD_OPT(Str,  processedimfileArgs, "-class_id",  "search by class ID",                    NULL);
+    ADD_OPT(Str,  processedimfileArgs, "-filter",  "search by filter",                        NULL);
     ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL);
     ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL);
+    ADD_OPT(S64,  processedimfileArgs, "-magicked",    "search by magicked value",            0);
+    ADD_OPT(Bool, processedimfileArgs, "-destreaked",   "only return imfiles that have been destreaked", false);
+    ADD_OPT(Bool, processedimfileArgs, "-not_destreaked", "only return imfiles that have not been destreaked", false);
     ADD_OPT(U64,  processedimfileArgs, "-limit",     "limit result set to N items",           0);
     ADD_OPT(Bool, processedimfileArgs, "-faulted",   "only return imfiles with a fault status set", false);

branches/eam_branches/20090715/ippTools/src/stacktool.c

@@ -859 +859 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
     PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "==");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
+
+//  The following three selectors are incompatible with the sql so omit them
+//    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
+//     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
+//    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -879 +886 @@
         psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
-    }
+    } else if (!all) {
+        psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required");
+        return false;
+    }
+
     psFree(where);
 

branches/eam_branches/20090715/ippTools/src/stacktoolConfig.c

@@ -164 +164 @@
     psMetadata *sumskyfileArgs= psMetadataAlloc();
     psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "search by stack ID", 0);
-    psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warp ID", 0);
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", 0);
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,         "search by skycell ID", 0);
+#ifdef notdef
+    // These don't work so omit (for now) We probably should create a different mode for this type of search.
+    psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warp ID", 0);
     psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0,            "search by exposure ID", 0);
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure name", NULL);
+#endif
+    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-label", 0,             "search by stackRun.label", NULL);
+    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-filter", 0,            "search by filter (LIKE comparison)", NULL);
+    psMetadataAddS16(sumskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
     psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
+    psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-all",  0,            "enable search without arguments", false);
 
     // -revertsumskyfile

branches/eam_branches/20090715/ippTools/src/warptool.c

@@ -1198 +1198 @@
     PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
     PXOPT_COPY_S64(config->args, where, "-fake_id",    "fakeRun.fake_id", "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawExp.dateobs",  "<=");
+    PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "==");
+    PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "==");
+    pxAddLabelSearchArgs (config, where, "-label",   "warpRun.label", "LIKE");
+
+    PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args,     "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false);
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -1213 +1224 @@
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
+    } else if (!all) {
+        psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required");
+        return false;
     }
     psFree(where);
+
+    if (not_destreaked) {
+        if (destreaked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense");
+            return false;
+        }
+        if (magicked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense");
+            return false;
+        }
+        psStringAppend(&query, " AND warpSkyfile.magicked = 0");
+    }
+    if (destreaked) {
+        psStringAppend(&query, " AND warpSkyfile.magicked != 0");
+    }
 
     // treat limit == 0 as "no limit"

branches/eam_branches/20090715/ippTools/src/warptoolConfig.c

@@ -215 +215 @@
     psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure tag", 0);
     psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0,           "search by phase 3 version of exposure tag", 0);
+    psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0,    "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_end", 0,      "search for exposures by time (<=)", NULL);
+    psMetadataAddStr(warpedArgs, PS_LIST_TAIL,  "-filter", 0,           "search for exposures by filter", NULL);
+    psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-destreaked",  0,      "search for destreaked images", false);
+    psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-not_destreaked",  0,  "search for images that have not been destreaked", false);
+    psMetadataAddS64(warpedArgs, PS_LIST_TAIL,  "-magicked", 0,         "search by magicked value", 0);
+    psMetadataAddS16(warpedArgs, PS_LIST_TAIL,  "-fault",  0,           "search by fault code", 0);
+    psMetadataAddStr(warpedArgs, PS_LIST_TAIL,  "-label",  PS_META_DUPLICATE_OK, "search by warpRun label", NULL);
+
+    psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
     psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);

branches/eam_branches/20090715/ppStack/src/ppStackLoop.c

@@ -80 +80 @@
         return false;
     }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 2: Generate Convolutions and Save: %f sec",
-             psTimerClear("PPSTACK_STEPS"));
     ppStackMemDump("convolve");
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 4: Make Initial Stack: %f sec", psTimerClear("PPSTACK_STEPS"));
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 3: Make Initial Stack: %f sec", psTimerClear("PPSTACK_STEPS"));
 
 
@@ -94 +92 @@
         return false;
     }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 5: Pixel Rejection: %f sec", psTimerClear("PPSTACK_STEPS"));
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 4: Pixel Rejection: %f sec", psTimerClear("PPSTACK_STEPS"));
     ppStackMemDump("reject");
 
@@ -106 +104 @@
         return false;
     }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 6: Final Stack: %f sec", psTimerClear("PPSTACK_STEPS"));
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 5: Final Stack: %f sec", psTimerClear("PPSTACK_STEPS"));
     ppStackMemDump("final");
 
@@ -118 +116 @@
         return false;
     }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 7: WCS & JPEGS: %f sec", psTimerClear("PPSTACK_STEPS"));
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 6: WCS & JPEGS: %f sec", psTimerClear("PPSTACK_STEPS"));
     ppStackMemDump("cleanup");
 
@@ -131 +129 @@
         return false;
     }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 8: Photometry Analysis: %f sec", psTimerClear("PPSTACK_STEPS"));
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 7: Photometry Analysis: %f sec", psTimerClear("PPSTACK_STEPS"));
     ppStackMemDump("photometry");
 
@@ -142 +140 @@
         return false;
     }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 9: Final output: %f sec", psTimerClear("PPSTACK_STEPS"));
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 8: Final output: %f sec", psTimerClear("PPSTACK_STEPS"));
     ppStackMemDump("finish");
 

branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.perl

@@ -3 +3 @@
   00    Module::Build                  Module-Build-0.2806.tar.gz               0.2806
   01    ExtUtils::MakeMaker            ExtUtils-MakeMaker-6.31.tar.gz           0
-  02    Params::Validate               Params-Validate-0.91.tar.gz       0.77
+  02    Params::Validate               Params-Validate-0.92.tar.gz              0.92
 #  02    Apache::Test                   Apache-Test-1.29.tar.gz                  1.29
   03    DateTime::TimeZone             DateTime-TimeZone-0.59.tar.gz            0
@@ -82 +82 @@
   78    SQL::Interp                     SQL-Interp-1.06.tar.gz
   79    Log::Dispatch::Email::MailSend  Log-Dispatch-2.22.tar.gz
-  80    Abstract::Meta::Class          Abstract-Meta-Class-0.11.tar.gz
+  80    Abstract::Meta::Class          Abstract-Meta-Class-0.13.tar.gz
   81    DBIx::Connection               DBIx-Connection-0.13.tar.gz
   82    Simple::SAX::Serializer        Simple-SAX-Serializer-0.05.tar.gz
   83    Test::Distribution             Test-Distribution-2.00.tar.gz
-  84    Test::DBUnit                   Test::DBUnit-0.20.tar.gz
+  84    Test::DBUnit                   Test-DBUnit-0.20.tar.gz                 0.20

branches/eam_branches/20090715/pstamp/scripts/Makefile.am

@@ -11 +11 @@
         pstamp_parser_run.pl \
         pstamp_queue_requests.pl \
+        pstamp_request_file \
         pstamp_results_file.pl \
         pstamp_revert_request.pl \

branches/eam_branches/20090715/pstamp/scripts/pstamp_finish.pl

@@ -81 +81 @@
     my $fileset = $req_name;
 
-
-    # Here we invoke the assumption that the output for the request is placed in the
-    # fileset directory directly
-#    my $out_dir = "$outputDataStoreRoot/$product/$fileset";
-
-    # now we are assuming that the output directory is the dirname of the request file
-    # XXX: put this in the database
-
     print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
 
@@ -216 +208 @@
 
                     # ra_deg and dec_deg are the coordinates of center of the stamp
-                    # XXX: parse the stamp header to find it
-                    print $tdf "0.0|0.0|";
+                    # XXX do this more cleanly
+                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields -x -1 RA_DEG DEC_DEG`;
+                    $ra_deg = 0.0 if (!$ra_deg);
+                    $dec_deg = 0.0 if (!$dec_deg);
+                    print $tdf "$ra_deg|$dec_deg|";
 
                     print $tdf "$exp_info|";
@@ -302 +297 @@
 
     if ($rownum eq 0) {
-        my $dummy_rowinfo = "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|";
+        my $dummy_rowinfo = "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|";
         return (undef, $dummy_rowinfo, "none");
     }
@@ -313 +308 @@
     my $tess_id = $row->{TESS_ID};
     $tess_id = "null" if !$tess_id;
+    my $comment = $row->{COMMENT};
+    $comment = "null" if !$comment;
+    my $label = $row->{LABEL};
+    $label = "null" if !$label;
 
     # This is ugly, error prone and hard to change.
     # Create a results file module and provide a list of the names (we have the data in the columns)
     my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|";
-    $rowinfo   .= "$row->{ID}|$tess_id|$component|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
+    $rowinfo   .= "$row->{ID}|$tess_id|$component|$label|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
     $rowinfo   .= "$row->{REQFILT}|$row->{COORD_MASK}|$row->{CENTER_X}|$row->{CENTER_Y}|";
-    $rowinfo   .= "$row->{WIDTH}|$row->{HEIGHT}";
+    $rowinfo   .= "$row->{WIDTH}|$row->{HEIGHT}|";
+    $rowinfo   .= $comment;
 
     return ($row, $rowinfo, $row->{PROJECT});

branches/eam_branches/20090715/pstamp/scripts/pstamp_job_run.pl

@@ -16 +16 @@
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use IPC::Cmd 0.36 qw( can_run run );
+use POSIX;
 
 use PS::IPP::Metadata::Config;
@@ -93 +94 @@
         run(command => $command, verbose => $verbose);
 
-    if ($success) {
+    my $exitStatus;
+    if (WIFEXITED($error_code)) {
+        $exitStatus = WEXITSTATUS($error_code);
+    } else {
+        print STDERR "ppstamp failed error_code: $error_code\n";
+        $exitStatus = $PS_EXIT_SYS_ERROR;
+    }
+
+    if ($exitStatus == 0) {
         my $dir = dirname($outputBase);
 
@@ -133 +142 @@
         close F;
         $jobStatus = $PS_EXIT_SUCCESS;
-    } else {
-        $jobStatus = $error_code >> 8;
-        my_die( "ppstamp failed with error code: $jobStatus", $job_id, $jobStatus);
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
+        $jobStatus = $PSTAMP_NO_OVERLAP
+    } else {
+        my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
     }
 } elsif ($jobType eq "get_image") {
@@ -162 +172 @@
 {
     my $command = "$pstamptool -updatejob -job_id $job_id -state stop"; 
+    $command .= " -fault $jobStatus" if $jobStatus;
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;

branches/eam_branches/20090715/pstamp/scripts/pstamp_results_file.pl

@@ -90 +90 @@
         { name => 'TESS_ID',    type => '64A', writetype => TSTRING },    
         { name => 'COMPONENT',  type => '64A', writetype => TSTRING },    
+        { name => 'LABEL',      type => '64A', writetype => TSTRING },    
 
         # output parameters
@@ -104 +105 @@
         { name => 'WIDTH',      type => 'D', writetype => TDOUBLE },   
         { name => 'HEIGHT',     type => 'D', writetype => TDOUBLE },  
+        { name => 'COMMENT',    type => '64A', writetype => TSTRING },    
 ];
 

branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl

@@ -16 +16 @@
 use File::Basename qw(basename);
 use Carp;
-use POSIX qw( strftime );
+use POSIX;
 
 my $verbose;
@@ -72 +72 @@
 }
 
+# list_job is a deugging mode
 $no_update = 1 if $mode eq "list_job";
 
@@ -185 +186 @@
     }
     my $req_type = $row->{REQ_TYPE};
-    $stage = $row->{IMG_TYPE};
-    my $id       = $row->{ID};
+    $stage       = $row->{IMG_TYPE};
+    my $id      = $row->{ID};
     my $component = $row->{COMPONENT};
-
-    my $filter   = $row->{REQFILT};
+    my $tess_id = $row->{TESS_ID};
+
+    my $filter  = $row->{REQFILT};
     my $mjd_min = $row->{MJD_MIN};
     my $mjd_max = $row->{MJD_MAX};
+    my $label   = $row->{LABEL};
 
     my $option_mask= $row->{OPTION_MASK};
@@ -222 +225 @@
     my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
 
-
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
@@ -228 +230 @@
 
     # Temporary hack so that MOPS can get at non-magicked data
-    if ($product eq "mops-pstamp-results") {
+    if ($product and ($product eq "mops-pstamp-results")) {
         $need_magic = 0;
     }
@@ -269 +271 @@
         my ($x, $y);
 
-        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $search_component,
-                $inverse, $skycenter, $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $tess_id, $search_component,
+                $inverse, $need_magic, $x, $y, $mjd_min, $mjd_max, $filter, $label, $verbose);
 
         if (!$imageList or !@$imageList) {
@@ -305 +307 @@
     my $num_jobs = 0;
     my $rownum = $row->{ROWNUM};
+    my $option_mask = $row->{OPTION_MASK};
     my $components = $row->{components};
 
@@ -346 +349 @@
         if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
             # XXX: should we add a faulted job so the client can know what happened if no images come back?
+            # This test is made in locate_images now so this code never runs. This leads to no feedback
+            # to users, but speeds up processing significatnly
             print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
 
@@ -369 +374 @@
         $args .= " -file $imagefile";
 
-        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
+        if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
             $args .= " -mask $image->{mask}";
         }
-        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
+        if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
             $args .= " -variance $image->{weight}";
         }
@@ -407 +412 @@
         $num_jobs++;
         my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
-                        . " -outputBase $output_base -rownum $rownum -state $newState";
+                        . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask";
         $command .= " -fault $fault" if $fault;
         $command .= " -exp_id $exp_id" if $exp_id;
@@ -542 +547 @@
                     run(command => $command, verbose => $verbose);
                 unless ($success) {
-                    print STDERR @$stderr_buf;
-                    my $rc = $error_code >> 8;
-                    my_die( "dvoImagesAtCoords failed: $rc", $PS_EXIT_UNKNOWN_ERROR);
+                    # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP
+                    # That just means that the coordinate didn't match any image/skycell
+                    if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) {
+                        print STDERR @$stderr_buf;
+                        my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR;
+                        my_die( "dvoImagesAtCoords failed: $rc", $rc);
+                    }
                 }
                 # now we have a list of row numbers and components
@@ -659 +668 @@
     return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
     return 0 if ($r1->{ID} ne $r2->{ID});
+    return 0 if ($r1->{TESS_ID} ne $r2->{TESS_ID});
+    return 0 if ($r1->{REQFILT} ne $r2->{REQFILT});
+    return 0 if ($r1->{LABEL} ne $r2->{LABEL});
+    return 0 if ($r1->{MJDMIN} ne $r2->{MJDMAX});
+    return 0 if ($r1->{MJDMAX} ne $r2->{MJDMAX});
     return 0 if ($r1->{inverse} ne $r2->{inverse});
 

branches/eam_branches/20090715/pstamp/src/Makefile.am

@@ -1 +1 @@
 bin_PROGRAMS = ppstamp pstamprequest pstampdump
+
+include_HEADERS = \
+        pstamp.h
 
 noinst_HEADERS = \
         ppstamp.h  \
+        pstampint.h \
         pstampErrorCodes.h
 

branches/eam_branches/20090715/pstamp/src/ppstamp.h

@@ -6 +6 @@
 #endif
 
-#include "pstamp.h"
+#include "pstampint.h"
 #include "ppstampOptions.h"
 

branches/eam_branches/20090715/pstamp/src/ppstampMakeStamp.c

@@ -48 +48 @@
 }
 
-static bool copyMetadata(pmFPAfile *output, pmFPAfile *input, pmChip *inChip, ppstampOptions *options)
+static bool copyMetadata(pmFPAfile *output, pmFPAfile *input, pmChip *inChip, ppstampOptions *options, pmAstromObj *center)
 {
     pmChip    *outChip;
@@ -61 +61 @@
     // since some of the keywords might be duplicated we may not want to copy both
 
-    // copy the fpa concepts
-    outChip->parent->concepts = psMetadataCopy(outChip->parent->concepts, inChip->parent->concepts);
-
     pmHDU *inHDU  = pmHDUFromChip(inChip);
     pmHDU *outHDU = pmHDUGetHighest(output->fpa, outChip, NULL);
@@ -73 +70 @@
     }
 
+    // copy the fpa concepts
+    pmConceptsCopyFPA(output->fpa, input->fpa, false, false);
+    pmConceptsCopyChip(outChip, inChip, false);
+    pmCell *outCell = outChip->cells->data[0]; // The only output cell
+
+    if (inChip->cells->n == 1) {
+        pmConceptsCopyCell(outCell, inChip->cells->data[0]);
+        // Need to fix up the trimsec and biassec to correspond to the output
+        psMetadataItem *trimsec = psMetadataLookup(outCell->concepts, "CELL.TRIMSEC");
+        psFree(trimsec->data.V);
+        trimsec->data.V = NULL;
+        psMetadataItem *biassec = psMetadataLookup(outCell->concepts, "CELL.BIASSEC");
+        psFree(biassec->data.V);
+        biassec->data.V = psListAlloc(NULL);
+    } else {
+        psList *inCells = psArrayToList(inChip->cells); // Input cells
+        pmConceptsAverageCells(outCell, inCells, NULL, NULL, false);
+        psFree(inCells);
+    }
 
     // If input had WCS convert it for stamp
@@ -93 +109 @@
     }
 
+    psMetadataAddF64(outHDU->header, PS_LIST_TAIL, "RA_DEG", PS_META_REPLACE, "Right Ascension of stamp center", RAD_TO_DEG(center->sky->r));
+    psMetadataAddF64(outHDU->header, PS_LIST_TAIL, "DEC_DEG", PS_META_REPLACE, "Declination of stamp center", RAD_TO_DEG(center->sky->d));
+
     ppstampVersionMetadata(outHDU->header, options);
 
@@ -178 +197 @@
 
 static int makeStamp(pmConfig *config, ppstampOptions *options, pmFPAfile *input,
-                pmChip *inChip, pmFPAview *view)
+                pmChip *inChip, pmFPAview *view, pmAstromObj *center)
 {
     int status = false;
@@ -285 +304 @@
 
     if (status) {
-        status = copyMetadata(output, input, inChip, options);
+        status = copyMetadata(output, input, inChip, options, center);
     }
     return status ? PS_EXIT_SUCCESS : PS_EXIT_UNKNOWN_ERROR;
@@ -360 +379 @@
 {
     pmAstromObj *pt = pmAstromObjAlloc();
-
-    if (!options->roip.celestialCenter) {
-        // Center was specified in chip coordinates, transform to the sky and the TP
-        chipToSky(center, fpa, chip);
-    }
 
     // calculate the four corners of the bounding box in sky coordinates, translate them to
@@ -470 +484 @@
             center->chip->yErr = 0;
             onChip = true;
+            chipToSky(center, input->fpa, chip);
         }
     }
@@ -573 +588 @@
         case PPSTAMP_ON:
         case PPSTAMP_PARTIALLY_ON:
-            returnval = makeStamp(config, options, input, chip, view);
+            returnval = makeStamp(config, options, input, chip, view, center);
             allDone = true;
             foundOverlap = true;

branches/eam_branches/20090715/pstamp/src/pstamp.h

@@ -1 +1 @@
 #ifndef PSTAMP_H
 #define PSTAMP_H
-
-#include <stdio.h>
-#include <string.h>
-#include <strings.h>  // for strcasecmp
-#include <unistd.h>   // for unlink
-#include "pslib.h"
-#include "psmodules.h"
-#include "pstampErrorCodes.h"
-
-typedef enum {
-    PSTAMP_UNKNOWN = -1,
-    PSTAMP_RAW,
-    PSTAMP_CHIP,
-    PSTAMP_WARP,
-    PSTAMP_DIFF,
-    PSTAMP_STACK
-} pstampImageType;
-
-
-// command modes for pstampparse 
-typedef enum {
-    PSP_MODE_UNKNOWN = 0,
-    PSP_MODE_QUEUE_JOB,
-    PSP_MODE_LIST_URI,
-    PSP_MODE_LIST_JOB
-} pspMode;
 
 // error codes returned to users in results flie

branches/eam_branches/20090715/pstamp/src/pstampGetROI.c

@@ -3 +3 @@
 #endif
 
-#include "pstamp.h"
+#include "pstampint.h"
 #include "pstampROI.h"
 #include "ohana.h"

branches/eam_branches/20090715/pstamp/src/pstamprequest.c

@@ -1 +1 @@
 #include <pslib.h>
 #include <string.h>
-#include "pstamp.h"
+#include "pstampint.h"
 #include "pstampROI.h"
 

branches/eam_branches/20090715/pstamp/test/asteroid.txt

@@ -16 +16 @@
 #
 
-
 # These coordinates get stamps from warp and diff images that show a moving object that mops found.
 
-# ROWNUM PROJECT       JOB_TYPE OPTION_MASK REQ_TYPE IMG_TYPE    ID      TESS_ID COMPONENT COORD_MASK CENTER_X   CENTER_Y             WIDTH HEIGHT  REQFILT MJD_MIN MJD_MAX
-
-1        gpc1           stamp     3         bydiff   warp      362483    NULL      NULL      2       284.688522833822 -3.86916140936841 200  200     null   0         0
-2        gpc1           stamp     3         bydiff   diff      362483    NULL      NULL      2       284.688522833822 -3.86916140936841 200  200     null   0         0
-3        gpc1           stamp     1027      bydiff   warp      362483    NULL      NULL      2       284.688522833822 -3.86916140936841 200  200     null   0         0
-4        gpc1           stamp     1027      bydiff   diff      362483    NULL      NULL      2       284.688522833822 -3.86916140936841 200  200     null   0         0
+# ID    |     ROI Specification                   |  JOB Specification | Images of interest specification
+# ROWNUM CENTER_X         CENTER_Y         WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT REQ_TYPE IMG_TYPE    ID   TESS_ID COMPONENTLABEL REQFILT MJD_MIN MJD_MAX | COMMENT
+#
+1        284.688522833822 -3.86916140936841 200  200        2       stamp    3           gpc1     bydiff   warp      362483  NULL    NULL   null  null    0       0      | 
+2        284.688522833822 -3.86916140936841 200  200        2       stamp    3           gpc1     bydiff   diff      362483  NULL    NULL   null  null    0       0      | 
+3        284.688522833822 -3.86916140936841 200  200        2       stamp    1027        gpc1     bydiff   warp      362483  NULL    NULL   null  null    0       0      | 
+4        284.688522833822 -3.86916140936841 200  200        2       stamp    1027        gpc1     bydiff   diff      362483  NULL    NULL   null  null    0       0      | 
 
 

branches/eam_branches/20090715/pstamp/test/gpc1_sample.txt

@@ -15 +15 @@
 # subsequent lines define the rows in the table
 
-# ROWNUM PROJECT JOB_TYPE OPTION_MASK REQ_TYPE IMG_TYPE    ID      TESS_ID COMPONENT   COORD_MASK CENTER_X   CENTER_Y WIDTH HEIGHT REQFILT MJD_MIN MJD_MAX
+## ID    |     ROI Specification                   |  JOB Specification | Images of interest specification 
+# ROWNUM CENTER_X       CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT REQ_TYPE IMG_TYPE    ID   TESS_ID COMPONENT    LABEL REQFILT MJD_MIN MJD_MAX | COMMENT
 # warps from various epochs for one of the SN candidates. Specifiying the skycell speeds up processing
-1         gpc1   stamp      1        byid        warp     6245   null skycell.077 2 242.400666 55.273513 200 200 null 0 0
-2         gpc1   stamp      1        byid        warp     6254   null skycell.077 2 242.400666 55.273513 200 200 null 0 0
-3         gpc1   stamp      1        byid        warp     6264   null skycell.077 2 242.400666 55.273513 200 200 null 0 0
-4         gpc1   stamp      1        byid        warp     6317   null skycell.077 2 242.400666 55.273513 200 200 null 0 0
-5         gpc1   stamp      1        byid        warp     6324   null skycell.077 2 242.400666 55.273513 200 200 null 0 0
-6         gpc1   stamp      1        byid        warp     6465   null skycell.077 2 242.400666 55.273513 200 200 null 0 0
-7         gpc1   stamp      1        byid        warp     6466   null skycell.077 2 242.400666 55.273513 200 200 null 0 0
-
+1        242.400666     55.273513 200   200     2          stamp    1          gpc1   byid      warp     6245   null    skycell.077   null null     0        0     |
+2        242.400666     55.273513 200   200     2          stamp    1          gpc1   byid      warp     6254   null    skycell.077   null null     0        0     |
+3        242.400666     55.273513 200   200     2          stamp    1          gpc1   byid      warp     6264   null    skycell.077   null null     0        0     |
+4        242.400666     55.273513 200   200     2          stamp    1          gpc1   byid      warp     6317   null    skycell.077   null null     0        0     |
+5        242.400666     55.273513 200   200     2          stamp    1          gpc1   byid      warp     6324   null    skycell.077   null null     0        0     |
+6        242.400666     55.273513 200   200     2          stamp    1          gpc1   byid      warp     6465   null    skycell.077   null null     0        0     |
+7        242.400666     55.273513 200   200     2          stamp    1          gpc1   byid      warp     6466   null    skycell.077   null null     0        0     |
 
 # get a specific warp
-10        gpc1   stamp      1        byid         warp     6316  null skycell.077 2 242.400666 55.273513 200 200 null 0 0
+10        242.400666     55.273513 200   200     2          stamp    1         gpc1   byid      warp     6316   null     skycell.077   null null     0        0     |
 
 # get stamps from all warps for exposure (only destreaked ones will succeed)
-11        gpc1   stamp      1        byexp        warp    o4973g0123o null skycell.077 2 242.400666 55.273513 200 200 null 0 0
+11        242.400666     55.273513 200   200     2          stamp    1         gpc1   byexp     warp  o4973g0123o   null skycell.077   null null     0        0     |
 
 # get stamps from all chipRuns for exposure (only destreaked ones will succeed)
 # add the mask and weight images as well
-12        gpc1   stamp      7        byexp      chip    o4973g0123o null null  2 242.400666 55.273513 200 200 null 0 0
+12        242.400666     55.273513 200   200     2          stamp    7         gpc1   byexp  chip  o4973g0123o   null    skycell.077   null null     0        0     |
 
 # get the corresponding diff
-13        gpc1   stamp      1        byexp      diff    o4973g0123o null null 2 242.400666 55.273513 200 200 null 0 0
+13        242.400666     55.273513 200   200     2          stamp    1        gpc1   byexp   diff  o4973g0123o   null    null         null null     0        0     |
 
-# get the stack that was the templae for that stack
-14        gpc1   stamp      1        byid     stack    14031  null null 2 242.400666 55.273513 200 200 null 0 0
-#
+# get the stack that was the template for that stack
+14        242.400666     55.273513 200   200     2          stamp    1         gpc1   byid    stack      14031   null     null        null null     0        0     |
+
 # get the same stamp by go through the diff
-15        gpc1   stamp      1        bydiff     stack    193939 null null 2 242.400666 55.273513 200 200 null 0 0
-
+15        242.400666     55.273513 200   200     2          stamp    1         gpc1   bydiff   stack    193939   null     null        null null     0        0     |

branches/eam_branches/20090715/pstamp/test/pstamp_req_create

@@ -1 +1 @@
 #!/usr/bin/env perl
+
+print STDERR "This script is obsolete. See ../scripts/pstamp_request_file\n";
+exit 1;
 
 # create a Postage Stamp Request file from a textual description

branches/eam_branches/20090715/pswarp/src/pswarpDefine.c

@@ -65 +65 @@
         int numCols = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); ///< Number of columns
         int numRows = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); ///< Number of rows
+        if ((numCols == 0) || (numRows == 0)) {
+            psError(PS_ERR_UNKNOWN, false, "skycell has invalid dimensions %d x %d", numCols, numRows);
+            psFree(view);
+            return false;
+        }
 
         pmCell *target = pmFPAviewThisCell(view, output->fpa); ///< Target cell

branches/eam_branches/20090715/pswarp/src/pswarpLoop.c

@@ -397 +397 @@
         }
 
+        pmModelClassSetLimits(PM_MODEL_LIMITS_STRICT);
+
         // measure the PSF using these sources
         if (!psphotReadoutFindPSF(config, view, sources)) {