Changeset 35412

Timestamp:

Apr 19, 2013, 3:39:39 PM (13 years ago)

Author:

eugene

Message:

merged from trunk

Location:

branches/eam_branches/ipp-20130307

Files:

• Ohana (modified) (1 prop)
• Ohana/src/libohana/src (modified) (1 prop)
• Ohana/src/opihi (modified) (1 prop)
• Ohana/src/opihi/cmd.astro (modified) (1 prop)
• Ohana/src/opihi/cmd.data (modified) (1 prop)
• Ohana/src/relastro/src (modified) (1 prop)
• ippScripts/scripts/camera_exp.pl (modified) (1 diff)
• ippScripts/scripts/chip_imfile.pl (modified) (1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/relgroup_exp_list.pl (modified) (6 diffs)
• ippScripts/scripts/warp_skycell.pl (modified) (1 prop)
• ippTools/share (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/pstamptool_completedreq.sql (modified) (1 diff)
• ippTools/share/pstamptool_getdependent.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 prop)
• ippTools/share/releasetool_listrelexp.sql (modified) (1 diff)
• ippTools/share/releasetool_listrelstack.sql (copied) (copied from trunk/ippTools/share/releasetool_listrelstack.sql )
• ippTools/share/releasetool_pendingrelgroup.sql (modified) (1 diff)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/camtool.c (modified) (1 prop)
• ippTools/src/camtoolConfig.c (modified) (1 prop)
• ippTools/src/magictool.c (modified) (1 prop)
• ippTools/src/pstamptool.c (modified) (12 diffs)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff)
• ippTools/src/releasetool.c (modified) (9 diffs)
• ippTools/src/releasetoolConfig.c (modified) (3 diffs)
• psLib/src/math/Makefile.am (modified) (2 diffs)
• psLib/src/math/psMixtureModels.c (copied) (copied from trunk/psLib/src/math/psMixtureModels.c )
• psLib/src/math/psMixtureModels.h (copied) (copied from trunk/psLib/src/math/psMixtureModels.h )
• psLib/src/pslib_strict.h (modified) (1 diff)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/psmkreq (modified) (13 diffs)
• pstamp/scripts/psstatus (modified) (6 diffs)
• pstamp/scripts/pstamp_cleanup.pl (modified) (1 diff)
• pstamp/scripts/pstamp_finish.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_request_file (modified) (4 diffs)
• pstamp/scripts/pstamp_webrequest.pl (modified) (6 diffs)
• pstamp/scripts/pstampparse.pl (modified) (34 diffs)
• pstamp/src/ppstampParseCamera.c (modified) (3 diffs)
• tools/heathers_plots/deploy.sh (copied) (copied from trunk/tools/heathers_plots/deploy.sh )
• tools/heathers_plots/heathers_plots.py (modified) (8 diffs)

branches/eam_branches/ipp-20130307/ippScripts/scripts/camera_exp.pl

@@ -363 +363 @@
     }
     # Construct FPA continuity corrected background images
-    {
+    if ($camera =~ /ISP/) {
+        print "Skipping FPA continuity corrected background images for ISP\n";
+    } else {
         my $command;
         $command = "$ppImage";

branches/eam_branches/ipp-20130307/ippScripts/scripts/relgroup_exp_list.pl

@@ -16 +16 @@
 
 
-my ($group_id, $group_type, $lap_id, $release_name, $rebuild, $dbname, $no_update, $verbose);
+my ($group_id, $group_type, $lap_id, $group_name, $release_name, $rebuild, $dbname, $no_update, $verbose);
 
 GetOptions(
     'group_id=s'    =>      \$group_id,
+    'group_type=s'  =>      \$group_type,
     'lap_id=s'      =>      \$lap_id,
-    'group_type=s'  =>      \$group_type,
+    'group_name=s'  =>      \$group_name,
     'release_name=s' =>     \$release_name,
     'rebuild'       =>      \$rebuild,      # rebuild directory if it exists
@@ -36 +37 @@
            -exitval => 3) 
         unless $lap_id;
+    $group_name = "lap_$lap_id";
+} else {
+    pod2usage( -msg => "--group_name is required with --group_type $group_type",
+           -exitval => 3) 
+        unless $group_name;
 }
 
@@ -48 +54 @@
 
 # XXX: get this from site.config
-my $outputBase = "/data/ippc17.0/www/ipp-misc/ps1-data/single-frame/$release_name";
-
-my $outdir;
-if ($group_type eq 'lap') {
-    $outdir = "$outputBase/lap_$lap_id";
-    if (-e $outdir) {
-        if ($rebuild) {
-            my $rc = system "rm -rf $outdir";
-            if ($rc) {
-                my $status = $rc >> 8;
-                my_die("failed to remove existing directory $outdir: $rc $status", $group_id, $PS_EXIT_UNKNOWN_ERROR);
-            }
-        } else {
-            my_die("directory $outdir already exists and --rebuild not supplied", $group_id, $PS_EXIT_UNKNOWN_ERROR);
-        }
-    }
-} else {
-    my_die("Not ready to do group_type $group_type\n", $group_id, $PS_EXIT_UNKNOWN_ERROR);
+my $exposureListRoot = "/data/ippc17.0/www/ipp-misc/ps1-data/single-frame";
+
+my $outputBase = "$exposureListRoot/$release_name";
+
+if (!-e $outputBase) {
+    mkdir $outputBase or &my_die("Failed to create $outputBase", $group_id, $PS_EXIT_SYS_ERROR);
+}
+
+my $outdir = "$outputBase/$group_name";
+if (-e $outdir) {
+    if ($rebuild) {
+        my $rc = system "rm -rf $outdir";
+        if ($rc) {
+            my $status = $rc >> 8;
+            my_die("failed to remove existing directory $outdir: $rc $status", $group_id, $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        my_die("directory $outdir already exists and --rebuild not supplied", $group_id, $PS_EXIT_UNKNOWN_ERROR);
+    }
 }
 
 mkdir $outdir or my_die ("failed to create $outdir", $group_id, $PS_EXIT_UNKNOWN_ERROR);
-
-if (0) {
-    # XXX: old code from prototype for nightly lists
-        my $outroot;
-        my $oh;
-        my ($year, $month, $day);
-        if ($outroot) {
-        die "usage: $0 <year> <month> <day>\n" unless $year and $month and $day;
-            my $monthdir = sprintf "$outroot/%4d/%02d", $year, $month;
-            if (! -e $monthdir) {
-                die "can't find directory for $year-$month\n";
-            }
-            my $daydir = sprintf "$monthdir/%02d", $day;
-            if (! -e $daydir) {
-                mkdir $daydir or die "failed to create $daydir\n";
-            }
-            my $outfile = sprintf "$daydir/index.html", $day;
-            open $oh, ">$outfile" or die "failed to open $outfile";
-        } else {
-            $oh = *STDOUT;
-        }
-
-        my $ticks = timegm(0, 0, 0, $day, $month - 1, $year - 1900);
-
-        my $mjd = 40587 + ($ticks / 86400.);
-        my $date = sprintf "%04d-%02d-%02d", $year, $month, $day;
-}
 
 my $oh;
@@ -105 +85 @@
 open $oh2, ">$outfile2" or my_die("failed to open $outfile2", $group_id, $PS_EXIT_UNKNOWN_ERROR);
 
-my $title = "PS1 Exposure List for $release_name LAP run $lap_id";
+my $title = "PS1 Exposure List for";
+if ($group_type eq 'lap') {
+    $title .= " $release_name LAP run $lap_id";
+} else {
+    $title .= " $group_name";
+}
 
 my $head = "
@@ -125 +110 @@
 
 
-# find all exposures on given day that have a chip run
+# find all exposures in this relGroup
 my $query = "
 SELECT 
@@ -149 +134 @@
 $stmt->execute() or my_die("failed to execute query: $stmt->errstr\n", $group_id, $PS_EXIT_UNKNOWN_ERROR);
 
-printf STDERR "%4d exposures in group for LAP run $lap_id\n", $stmt->rows;
+printf STDERR "%4d exposures in group for $group_name\n", $stmt->rows;
 
 # $stmt->dump_results(1000);

branches/eam_branches/ipp-20130307/ippTools/share/Makefile.am

@@ -473 +473 @@
         releasetool_definerelstack.sql \
         releasetool_definerelstack_with_skycal.sql \
+        releasetool_listrelstack.sql \
         releasetool_definerelgroup_select_lap.sql \
         releasetool_definerelgroup_select_data_group.sql \

branches/eam_branches/ipp-20130307/ippTools/share/pstamptool_completedreq.sql

@@ -8 +8 @@
         SELECT count(*) FROM pstampJob 
         WHERE pstampJob.req_id = pstampRequest.req_id
-                AND pstampJob.state != 'stop'
+                AND pstampJob.state != 'stop' 
+                AND pstampJob.state != 'cancel'
         ) = 0

branches/eam_branches/ipp-20130307/ippTools/share/pstamptool_getdependent.sql

@@ -1 +1 @@
 SELECT DISTINCT pstampDependent.*
 FROM pstampDependent
-WHERE pstampDependent.state = 'new'
+WHERE (pstampDependent.state = 'new' OR pstampDependent.state = 'hold')

branches/eam_branches/ipp-20130307/ippTools/share/releasetool_listrelexp.sql

@@ -25 +25 @@
     warpRun.warp_id,
     warpRun.state as warp_state,
+    warpRun.tess_id,
     rawExp.filter,
     rawExp.dateobs,

branches/eam_branches/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql

@@ -5 +5 @@
     JOIN ippRelease USING(rel_id)
 WHERE relGroup.state = 'new'
+    AND relGroup.fault = 0

branches/eam_branches/ipp-20130307/ippTools/src/pstamptool.c

@@ -49 +49 @@
 static bool revertjobMode(pxConfig *config);
 
-# if (1)
-// these are unused functions -- since they are 'static', this raises an warning
-// XXX: They actually are used
 static bool addprojectMode(pxConfig *config);
 static bool projectMode(pxConfig *config);
@@ -63 +60 @@
 static bool listfileMode(pxConfig *config);
 static bool deletefileMode(pxConfig *config);
-# endif
 
 # define MODECASE(caseName, func) \
@@ -103 +99 @@
         MODECASE(PSTAMPTOOL_MODE_STOPDEPENDENTJOB, stopdependentjobMode);
         MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode);
-# if (1)
         MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode);
         MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode);
@@ -115 +110 @@
         MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode);
         MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode);
-# endif
+
         default:
             psAbort("invalid option (this should not happen)");
@@ -624 +619 @@
     if (outdir) {
         psStringAppend(&query, ", outdir = '%s'", outdir);
+    }
+    if (username) {
+        psStringAppend(&query, ", username = '%s'", username);
     }
     if (clearfault) {
@@ -1236 +1234 @@
     PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false);
     PXOPT_LOOKUP_BOOL(no_create,  config->args, "-no_create", false);
+    PXOPT_LOOKUP_BOOL(hold,       config->args, "-hold", false);
 
     psMetadata *where = psMetadataAlloc();
@@ -1243 +1242 @@
     PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
 
-    // start a transaction eraly so it will contain any row level locks
+    // start a transaction early so it will contain any row level locks
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1267 +1266 @@
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1274 +1276 @@
     if (!output) {
         psError(PS_ERR_UNKNOWN, false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1282 +1287 @@
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
         }
@@ -1287 +1295 @@
         if (fault > 0) {
             fprintf(stderr, "existing dependent has fault %d\n", fault);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            // return the fault to the client. This is used by the postage stamp parser
             exit (fault);
         }
+        bool commit = false;
+        // Check the state of the exisiting dependent. By query it is either
+        // new or hold
+        psString state = psMetadataLookupStr(NULL, dep, "state");
+        if (!hold && !strcmp(state, "hold")) {
+            // There is a dependent for this component but it's state is hold.
+            // This client needs one that will run.
+            // Update the state
+            psString updateQuery = NULL;
+            psStringAppend(&updateQuery, "UPDATE pstampDependent SET state = 'new' "
+                "\nWHERE stage = '%s' AND imagedb = '%s' AND stage_id = %"PRId64 " AND component = '%s'", 
+                    stage, imagedb, stage_id, component);
+
+            if (!p_psDBRunQuery(config->dbh, updateQuery)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(updateQuery);
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                return false;
+            }
+            // set flag to commit this change
+            commit = true;
+            psFree(updateQuery);
+        }
+        // print the dep_id for the user
         printf("%" PRId64 "\n", dep_id);
         psFree(output);
-        if (!psDBRollback(config->dbh)) {
+        // now either commit the change or rollback the transaction which releases the lock
+        if (commit) {
+            if (!psDBCommit(config->dbh)) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
             psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+            }
+        } else if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
         }
         return true;
@@ -1310 +1359 @@
         config->dbh,
         0,              // dep_id
-        "new",          // state
+        hold ? "hold" : "new",          // state
         stage,
         stage_id,

branches/eam_branches/ipp-20130307/ippTools/src/pstamptoolConfig.c

@@ -223 +223 @@
     psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-need_magic", 0, "define need_magic", false);
     psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-outdir", 0,    "define output directory for dependent (required)", NULL);
+    psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-hold", 0, "if creating new dependent set it's state to hold", false);
     psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no matching dependent do not create one", false);
 

branches/eam_branches/ipp-20130307/ippTools/src/releasetool.c

@@ -832 +832 @@
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
     PXOPT_LOOKUP_BOOL(simple, config->args,  "-simple", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 
     // find the parameters of all the exposures that we want to add to the release
@@ -852 +853 @@
 
     psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
 
     if (!p_psDBRunQuery(config->dbh, query)) {
@@ -968 +975 @@
     psMetadata *where = psMetadataAlloc();
 
+    PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-stack_id",    "relStack.stack_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-skycal_id",   "relStack.skycal_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "==");
     PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE");
     pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "==");
-    PXOPT_COPY_STR(config->args, where, "-state",       "relExp.state", "==");
-    PXOPT_COPY_STR(config->args, where, "-filter",      "rawExp.filter", "LIKE");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">=");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_major", ">=");
+    PXOPT_COPY_STR(config->args, where, "-state",       "relStack.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter",      "relStack.filter", "LIKE");
+    PXOPT_COPY_F32(config->args, where, "-mjd_min",    "stackSumSkyfile.mjd_obs", ">=");
+    PXOPT_COPY_F32(config->args, where, "-mjd_max",    "stackSumSkyfile.mjd_obs", "<=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_minor", ">=");
     PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
-    PXOPT_COPY_STR(config->args, where, "-exp_name",    "rawExp.exp_ame", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id",      "relExp.exp_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-chip_id",     "relExp.chip_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-cam_id",      "relExp.cam_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-warp_id",     "warpRun.warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-cam_data_group",  "camRun.data_group", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",     "relStack.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "relStack.skycell_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-stack_data_group",  "stackRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
 
     PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
     PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
+    pxskycellAddWhere(config, where);
 
     PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false);
@@ -993 +1001 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = pxDataGet("releasetool_listrelexp.sql");
+    psString query = pxDataGet("releasetool_listrelstack.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    psString where2 = NULL;
-    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
-        psError(psErrorCodeLast(), false, "pxspaceAddWhere failed");
         return false;
     }
@@ -1009 +1011 @@
         psStringAppend(&query, "\nWHERE %s", whereClause);
         psFree(whereClause);
-    } else if (where2) {
-        psStringAppend(&query, "\nWHERE ");
     } else {
         psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
@@ -1017 +1017 @@
     }
 
-    if (where2) {
-        psStringAppend(&query, "\n%s", where2);
-        psFree(where2);
-    }
-
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority");
+        psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority");
     }
 
@@ -1051 +1046 @@
     }
 
-    if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) {
+    if (!ippdbPrintMetadatas(stdout, output, "relStack", !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print array");
         psFree(output);
@@ -1331 +1326 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S32(group_id, config->args,      "-group_id", true, false);
+    PXOPT_LOOKUP_S32(group_id, config->args,      "-group_id", false, false);
+    PXOPT_LOOKUP_STR(group_name, config->args,    "-group_name",  false, false);
+    if (!group_id && !group_name) {
+        psError(PXTOOLS_ERR_CONFIG, true, "either group_id or group_name is required\n");
+        return false;
+    }
 
     PXOPT_LOOKUP_STR(state,    config->args,    "-set_state",  false, false);
@@ -1352 +1352 @@
 
     PXOPT_COPY_S64(config->args, where, "-group_id", "relGroup.group_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-group_name", "relGroup.group_name", "==");
     // XXX TODO if label is enabled (for changing label or state for a block) we should
     // disallow setting some parameters such as exp_list_path

branches/eam_branches/ipp-20130307/ippTools/src/releasetoolConfig.c

@@ -195 +195 @@
     psMetadataAddF32(definerelstackArgs,  PS_LIST_TAIL, "-set_zpt_stdev", 0,  "define zero point stdev", NAN);
     psMetadataAddF32(definerelstackArgs,  PS_LIST_TAIL, "-set_fwhm_major", 0, "define fwhm_major", NAN);
-    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-set_path_base", 0,  "define state (required)", NULL);
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-set_path_base", 0,  "define state", NULL);
     psMetadataAddS16(definerelstackArgs,  PS_LIST_TAIL, "-set_fault", 0,      "define fault", 0);
 
-    psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-pretend",  0, "do not actually modify the database", false);
+    psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false);
     psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+    psMetadataAddU64(definerelstackArgs, PS_LIST_TAIL,  "-limit",   0, "limit result set to N items", 0);
+
+    // -listrelstack
+    psMetadata *listrelstackArgs = psMetadataAlloc();
+    psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL,  "-relstack_id", 0,   "select by released exposure ID", 0);
+    psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL,  "-stack_id", 0,   "select by stack ID", 0);
+    psMetadataAddS64(listrelstackArgs, PS_LIST_TAIL,  "-skycal_id", 0,   "select by skycal ID", 0);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-release_state", PS_META_DUPLICATE_OK, "select by release state", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-state", 0,        "select by released stack state", NULL);
+
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-filter", 0,       "select by filter name (LIKE comparison)", NULL);
+    // psMetadataAddTime(listrelstackArgs, PS_LIST_TAIL, "-dateobs_begin", 0,"search for exposures by time (>=)", NULL);
+    // psMetadataAddTime(listrelstackArgs, PS_LIST_TAIL, "-dateobs_end", 0,  "search for exposures by time (<=)", NULL);
+
+    pxskycellAddArguments(listrelstackArgs);
+
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-fwhm_min", 0, "search by measured seeing (>=)", NAN);
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-fwhm_max", 0, "search by seeing (<=)", NAN);
+
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id (LIKE comparision)", NULL);
+
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-stack_data_group", 0, "select by stackRun.data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-skycal_data_group", 0, "select by skycalRun.data_group (LIKE comparison)", NULL);
+
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL);
+    psMetadataAddS64(listrelstackArgs,  PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0);
+
+    psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-priority_order",   0, "order by release priority", false);
+
+    psMetadataAddU64(listrelstackArgs,  PS_LIST_TAIL, "-limit",       0, "limit result set to N items", 0);
+    psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-simple",      0, "use the simple output format", false);
+
 
     // -definerelgroup
@@ -240 +274 @@
     psMetadata *updaterelgroupArgs = psMetadataAlloc();
 
-    psMetadataAddS32(updaterelgroupArgs,  PS_LIST_TAIL, "-group_id", 0,   "select by group ID (required)", 0);
+    psMetadataAddS32(updaterelgroupArgs,  PS_LIST_TAIL, "-group_id", 0,   "select by group ID",  0);
+    psMetadataAddStr(updaterelgroupArgs,  PS_LIST_TAIL, "-group_name", 0, "select by group name", 0);
 
         // parameters of the relGroup
@@ -272 +307 @@
 
     PXOPT_ADD_MODE("-definerelstack",     "define a released stack",    RELEASETOOL_MODE_DEFINERELSTACK,  definerelstackArgs);
+    PXOPT_ADD_MODE("-listrelstack",       "list released stacks",      RELEASETOOL_MODE_LISTRELSTACK,    listrelstackArgs);
 
     PXOPT_ADD_MODE("-definerelgroup",     "define a group of exposures", RELEASETOOL_MODE_DEFINERELGROUP,  definerelgroupArgs);

branches/eam_branches/ipp-20130307/psLib/src/math/Makefile.am

@@ -16 +16 @@
         psMinimizePowell.c \
         psMinimizePolyFit.c \
+        psMixtureModels.c \
         psPolynomial.c \
         psPolynomialMetadata.c \
@@ -45 +46 @@
         psMinimizePowell.h \
         psMinimizePolyFit.h \
+        psMixtureModels.h \
         psPolynomial.h \
         psPolynomialMetadata.h \

branches/eam_branches/ipp-20130307/psLib/src/pslib_strict.h

@@ -76 +76 @@
 #include "psMinimizePowell.h"
 #include "psMinimizePolyFit.h"
+#include "psMixtureModels.h"
 #include "psMutex.h"
 #include "psRandom.h"

branches/eam_branches/ipp-20130307/pstamp/scripts/psmkreq

@@ -28 +28 @@
 my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base);
 
-my ($image, $mask, $variance, $cmf, $psf, $backmdl, $inverse);
-my ($unconvolved, $use_imfile_id, $no_wait);
+my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse);
+my ($convolved, $unconvolved, $uncompressed, $use_imfile_id, $no_wait);
+
+# new header keywords for version 2
+my $action = 'PROCESS';
+my $email = 'null';
 
 my $default_size = 100;
@@ -52 +56 @@
 my $comment;
 
+# new request specification columns for version 2
+my $survey_name ='null';
+my $release_name = 'null';
+my $run_type ='null';
+my $fwhm_min = 0;
+my $fwhm_max = 0;
+
+
 my $missing_tools;
 my $pstamp_request_file  = can_run('pstamp_request_file')  or (warn "Can't find required program pstamp_request_file"  and $missing_tools = 1);
@@ -62 +74 @@
 GetOptions(
     'list=s'            => \$list,          # list of coordinates if undef ra and dec or x and y are required
-    'ra=s'              => \$ra,            # 
+    'ra=s'              => \$ra,
     'dec=s'             => \$dec,
     'x=s'               => \$x,
@@ -87 +99 @@
     'comment=s'         => \$comment,
 
+    'survey=s'          => \$survey_name,
+    'release=s'         => \$release_name,
+    'run_type=s'        => \$run_type,
+    'fwhm_min=s'        => \$fwhm_min,
+    'fwhm_max=s'        => \$fwhm_max,
+
     'option_mask=i'     => \$option_mask,
     'image'             => \$image,
     'mask'              => \$mask,
+    'jpeg'              => \$jpeg,
     'variance'          => \$variance,
     'cmf'               => \$cmf,
@@ -96 +115 @@
     'inverse'           => \$inverse,
     'unconvolved'       => \$unconvolved,
+    'convolved'         => \$convolved,
+    'uncompressed'      => \$uncompressed,
     'use_imfile_id'     => \$use_imfile_id,
     'do_not_wait'       => \$no_wait,
+
+    'action=s'          => \$action,
+    'email=s'           => \$email,
 
     'verbose'           => \$verbose,
@@ -156 +180 @@
 }
 
+if ($stage eq 'stack') {
+    # default is to do convolved stack
+    unless (defined $option_mask || $convolved) {
+        $unconvolved = 1;
+    }
+}
+
 checkFilter($filter, 'null', $filter)  if $filter;
 checkMJD($mjd_min, 0, "") if $mjd_min;
@@ -167 +198 @@
         $option_mask |= $PSTAMP_SELECT_MASK     if $mask;
         $option_mask |= $PSTAMP_SELECT_VARIANCE if $variance;
+        $option_mask |= $PSTAMP_SELECT_JPEG     if $jpeg;
+
+        # if no image was requested make a stamp of the image
         $option_mask = $PSTAMP_SELECT_IMAGE    if $option_mask == 0;
 
-        # if no image was requested make a stamp of the image
 
         $option_mask |= $PSTAMP_SELECT_CMF      if $cmf;
@@ -207 +240 @@
 } else {
     $rows = [];
-    push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max);
+    push @$rows, buildRow("", $comment, 1, $x, $y, $filter, $mjd_min, $mjd_max);
 }
 
 my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps);
-print $tdf "$req_name 1\n";
+my $line = "$req_name 2 $action $email";
+print "$line\n" if $verbose;
+print $tdf "$line\n";
 my $rownum = 0;
 foreach my $row (@$rows) {
     $rownum++;
     my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height"
-        . " $coord_mask $job_type $option_mask $project $req_type"
+        . " $coord_mask $job_type $option_mask $project $survey_name $release_name $req_type"
         . " $stage $id $tess_id $component $data_group"
-        . " $row->{filter} $row->{mjd_min} $row->{mjd_max}";
+        . " $row->{filter} $row->{mjd_min} $row->{mjd_max} $run_type $fwhm_min $fwhm_max";
 
     if ($row->{comment} and $row->{comment} ne '') {
@@ -235 +270 @@
     $command .= " --output $output" if $output;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+        run(command => $command, verbose => $verbose);
     unless ($success) {
         print STDERR @$stderr_buf;
@@ -274 +309 @@
 
     my $row = {};
-    $row->{ra}      = checkRA($vals[0], $linenumber);
-    $row->{dec}     = checkDEC($vals[1], $linenumber);
-    $row->{filter}  = checkFilter($vals[2], $filter, $linenumber);
-    $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);;
-    $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);;
+    $row->{rownum}  = checkRownum($vals[0], $linenumber);
+    $row->{ra}      = checkRA($vals[1], $linenumber);
+    $row->{dec}     = checkDEC($vals[2], $linenumber);
+    $row->{filter}  = checkFilter($vals[3], $filter, $linenumber);
+    $row->{mjd_min} = checkMJD($vals[4], $mjd_min, $linenumber);;
+    $row->{mjd_max} = checkMJD($vals[5], $mjd_max, $linenumber);;
     $row->{comment} = $comment;
 
@@ -305 +341 @@
     my $linenumber = shift;
 
-    my $result;
+   my $result;
     if ($c =~ /\:/) {
         # sexagesmial format not valid for pixel coordinates
@@ -329 +365 @@
 }
 
+sub checkRownum {
+    my $rownum = shift;
+    my $linenumber = shift;
+    die "rownum can not be null at $linenumber\n" unless defined $rownum;
+
+    # XXX: For now just check that rownum is defined and a single word
+    # extend this. Should we require a number?
+    my @words = split " ", $rownum;
+    die "$rownum is not a vaild rownum at $linenumber\n" unless (scalar @words) == 1;
+}
+
 sub checkRA {
     my $ra = shift;
@@ -335 +382 @@
 
     if ($ra =~ /\:/) {
+        # assume RA is in hours:minutes:seconds
         return $checked * 360. / 24.;
     } else {

branches/eam_branches/ipp-20130307/pstamp/scripts/psstatus

@@ -68 +68 @@
     IFNULL(depcount, 0) AS 'Image updates'
     , timestamp AS 'last state change (UTC)'
+    , registered AS 'Submit time (UTC)'
 FROM (
     SELECT
@@ -77 +78 @@
         pstampRequest.fault,
         timestamp,
+        registered,
         IFNULL(priority,10000) AS priority
 FROM pstampRequest
 LEFT JOIN Label USING(label)
     WHERE pstampRequest.state = 'run' or pstampRequest.state ='new' 
+        OR pstampRequest.state = 'parsed'
    --     OR pstampRequest.state = 'run.wait'
 ) as unfinishedReq
@@ -89 +92 @@
 FROM pstampRequest JOIN pstampJob USING(req_id)
     WHERE (pstampRequest.state = 'run' or pstampRequest.state = 'new' 
-        OR pstampRequest.state = 'run.wait')
+        OR pstampRequest.state = 'run.wait'
+        OR pstampRequest.state = 'parsed')
     GROUP BY req_id
 ) as rowCounts
@@ -101 +105 @@
     JOIN pstampDependent USING(dep_id) 
     WHERE (pstampRequest.state = 'run' or pstampRequest.state = 'new' 
+            OR pstampRequest.state = 'parsed'
             OR pstampRequest.state = 'run.wait')
         AND dep_id > 0
@@ -178 +183 @@
     IFNULL(faulted,0) AS 'Faulted Jobs',
     IFNULL(depcount, 0) AS 'Image updates completed',
-    timestamp AS 'Completion Time (UTC)'
+    timestamp AS 'Completion Time (UTC)',
+    registered AS 'Submit Time (UTC)'
 FROM (
     SELECT
@@ -187 +193 @@
 --    pstampRequest.state,
 --    pstampRequest.fault,
-    timestamp
+    timestamp,
+    registered
 FROM pstampRequest
     WHERE pstampRequest.state = 'stop'

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_cleanup.pl

@@ -183 +183 @@
     }
 
-    my $command = "rm -r $dir";
+    # do an ls of the directory before starting
+#    print "directory listing for $dir\n";
+#    system "ls $dir";
+
+    my $command = "rm -rf $dir";
     unless ($no_update) {
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
         unless ($success) {
-            my $rc = $error_code >> 8;
-            my_die("Unable to perform $command return status: $rc", $req_id, $rc);
+            if (-e $dir) {
+                my $rc = $error_code >> 8;
+                my_die("Unable to perform $command return status: $rc", $req_id, $rc);
+            } else {
+                print STDERR "rm claimed to fail but directory no longer exists\n";
+            }
         }
     } else {

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_finish.pl

@@ -107 +107 @@
     my ($header, $rows) = read_request_file($req_file);
 
-    my $action = $header->{ACTION};
-
-    if (!$header or (($action eq 'PROCESS') and !$rows)) {
+    if (!$header or !$rows) {
         # Since a request got queued, the request file must have been readable at some point 
         my_die("failed to read request file $req_file", $req_id, $PS_EXIT_CONFIG_ERROR);
@@ -128 +126 @@
 
     my $request_fault = 0;
-    if ($action eq 'PROCESS') {
+    {
         # The results table definition file
         my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
@@ -238 +236 @@
                 } else {
                     my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR) 
-                        if $fault eq $PSTAMP_SUCCESS;
+                        if $job->{state} eq 'stop' and $fault eq $PSTAMP_SUCCESS;
                     print STDERR "no reglist file for job $job_id\n" if $verbose;
                     print $tdf "$rownum|$fault|$error_string|0|$job_id|";
@@ -273 +271 @@
             }
         }
-    } elsif ($action ne 'LIST') {
-        my_die("Unexpected action $action found", $req_id, $PS_EXIT_PROG_ERROR);
-    } else {
-        # pstampparse did all of the work
     }
 

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_job_run.pl

@@ -113 +113 @@
 
     if ($stage eq "raw") {
+        # zap options that don't apply to raw stage
         $options &= ~($PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE); 
     }
@@ -136 +137 @@
         $argString .= " -astrom $params->{astrom}";
         push @file_list, $params->{astrom};
+    }
+
+    if ($options & $PSTAMP_SELECT_SOURCES) {
+        # Extract sources from astrometry file if provided. This will be the smf for chip stage
+        # or the skycal cmf for stacks
+        if ($params->{astrom}) {
+            $argString .= " -write_cmf";
+            if ($stage eq 'stack') {
+                # Set psphot recipe to STACKPHOT so that the extended source paramters will
+                # be copied from the cmf file.
+                $argString .= " -recipe PSPHOT STACKPHOT"
+            }
+        } elsif ($params->{cmf}) {
+            $argString .= " -write_cmf";
+            push @file_list, $params->{cmf};
+        } else {
+            print "Could not find suitable sources file will not write cmf\n";
+        }
     }
 
@@ -254 +273 @@
                            $PSTAMP_SELECT_MASK     => "mk.fits",
                            $PSTAMP_SELECT_VARIANCE => "wt.fits",
+                           $PSTAMP_SELECT_SOURCES  => "cmf",
                            $PSTAMP_SELECT_JPEG     => "jpg");
 
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG);
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
 
         foreach my $key (keys (%extensions)) {
@@ -400 +420 @@
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $cmf_file;
-        if ($stage ne 'chip') {
-            # we don't ship chip stage cmf files because they may not be censored
-            $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
-        }
+#        if ($stage ne 'chip') {
+#            # we don't ship chip stage cmf files because they may not be censored
+#            $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
+#        }
 
         my $outdir = dirname($output_base);

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_parser_run.pl

@@ -158 +158 @@
 my $request_fault = $PSTAMP_INVALID_REQUEST;
 
+# default action is to process the request after parsing. This can be overridden by
+# PREVIEW mode for pstamp requests
+my $action = 'PROCESS';
+
 if (-r $uri) {
     # run the appropriate parse command to parse the queue the jobs for this request
     # first check the extension header to find the EXTNAME
-    $request_type = find_request_type($uri);
+    $request_type = find_request_type($uri, \$action);
 
     if ($request_type) {
@@ -240 +244 @@
 
     if ($success) {
-        $newState = 'run';
+        # XXX: This bit of the postage stamp request API has slipped in here because we need to control
+        # the new state of the request
+        if ($action eq 'PROCESS') {
+            $newState = 'run';
+        } elsif ($action eq 'PREVIEW') {
+            $newState = 'parsed';
+        } else {
+            print STDERR "WARNING Ignoring unexpected value for ACTION found in request header: $action\n";
+        }
     } else {
         $fault = $error_code >> 8;
@@ -275 +287 @@
     # find the EXTNAME in the input fits table
     my $file_name = shift;
-    my $out = `echo $file_name | fields -x 0 EXTNAME`;
+    my $r_action = shift;
+
+    my $out = `echo $file_name | fields -x 0 EXTNAME ACTION`;
 
     if ($out) {
         # output from fields is filename value
-        my ($dummy, $extname) = split " ", $out;
+        my ($dummy, $extname, $action) = split " ", $out;
+
+        # Set the action if it is defined in the request header
+        # XXX:consider doing this only if extname is PS1_PS_REQUSET
+        $$r_action = $action if ($action);
 
         return $extname;

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_request_file

@@ -54 +54 @@
                     value => undef
         },
-        { name => 'USERNAME',
-                    writetype => TSTRING,
-                    comment => 'username for request (optional)',
-                    value => undef
-        },
+#        { name => 'USERNAME',
+#                    writetype => TSTRING,
+#                    comment => 'username for request (optional)',
+#                    value => undef
+#        },
         { name => 'EMAIL',
                     writetype => TSTRING,
@@ -97 +97 @@
         { name => 'MJD_MIN',    type => 'D',   writetype => TDOUBLE },
         { name => 'MJD_MAX',    type => 'D',   writetype => TDOUBLE },
+        # new in version 2
         { name => 'RUN_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'FWHM_MIN',   type => 'D',   writetype => TDOUBLE },
+        { name => 'FWHM_MAX',   type => 'D',   writetype => TDOUBLE },
 
         { name => 'COMMENT ',   type => '64A', writetype => TSTRING },
@@ -326 +329 @@
         pop @colData;
         pop @colData;
+        pop @colData;
+        pop @colData;
     }
 
@@ -346 +351 @@
         }
         # trim leading whitespace from comment
-        $comment =~ s/^\s+//;
+        $comment =~ s/^\s+// if $comment;
         if ($comment) {
             if ($comment =~ /#/) {

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_webrequest.pl

@@ -5 +5 @@
 #
 # The arguments are the command line parameters for the program psmkreq
-#
-# Unless the argument -list is provided the output is the request id for the resulting request
-#
-# If -list is the first argument, the request file is built and parsed and the
-# the selected "input" uris are listed on stdout.
 #
 # Note: Despite the name there nothing particularly web specific about this program.
@@ -23 +18 @@
 
 my $host = hostname();
-my $verbose = 1;
+my $verbose = 0;
 my $dbname;
 my $dbserver;
+my $pretend;
+my $label = 'WEB';
+my $req_name_prefix = 'web';
+my $username;
 
 GetOptions(
     'dbname=s'      =>  \$dbname,
     'dbserver=s'    =>  \$dbserver,
-    'verbose'       => \$verbose,
+    'label=s'       =>  \$label,
+    'username=s'    =>  \$username,
+    'prefix=s'      =>  \$req_name_prefix,
+    'pretend'       =>  \$pretend,
+    'verbose'       =>  \$verbose,
 );
 
@@ -36 +39 @@
 
 if ($verbose) {
-    print "\n\n";
-    print "Starting script $0 on $host\n\n";
+    print STDERR "\n\n";
+    print STDERR "Starting script $0 on $host\n\n";
 }
 
@@ -70 +73 @@
 # make a request file
 my $cur_dir = getcwd();
+
 #print STDERR "cur_dir is $cur_dir\n";
 
-my $datestr = strftime "%Y%m%d", gmtime;
+my $datestr = strftime "%Y/%m/%d", gmtime;
 my $datedir = "$cur_dir/webreq/$datestr";
 if (! -e $datedir ) {
-    if (!  mkdir $datedir ) {
-        print STDERR  "failed to create working directory $datedir";
+    my $rc = system "mkdir -p $datedir";
+    if ($rc) {
+        my $status = $rc >> 8;
+        print STDERR  "failed to create working directory $datedir: $rc $status";
         exit $PS_EXIT_CONFIG_ERROR;
     }
 }
 
-my $request_name = "web_" . get_webreq_num();
+my $request_name = $req_name_prefix . '_' . get_webreq_num();
 my $request_file = "$datedir/$request_name.fits";
 {
@@ -96 +102 @@
 # Queue the request
 my $req_id = 0;
-{
+unless ($pretend) {
 
     my $command = "$pstamptool -addreq -uri $request_file -ds_id 0";
-    $command .= " -label WEB";
+    $command .= " -label $label";
+    $command .= " -username $username" if $username;
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -109 +116 @@
     }
     $req_id = ${$stdout_buf}[0];
+    chomp $req_id;
 }
 

branches/eam_branches/ipp-20130307/pstamp/scripts/pstampparse.pl

@@ -27 +27 @@
 my $outdir;
 my $product;
-my $label;
+my $label = "";
 my $save_temps;
 my $no_update;
@@ -57 +57 @@
     die "outdir is required"  if !$outdir;
     die "product is required"  if !$product;
+} else {
+    $req_id = 0;
+    $outdir = "nowhere";
+    $product = "dummy";
 }
 
@@ -83 +87 @@
 $pstamptool .= " -dbserver $dbserver" if $dbserver;
 
-# list_job is a deugging mode
-$no_update = 1 if $mode eq "list_job";
+# If $mode is not queue_job we are using a debugging mode
+# do not update the database
+$no_update = 1 if $mode ne "queue_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 #
-# get the data from the extension header
+# Read the keywords from the extension header
 #
 my $fields_output;
 {
-    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USER EMAIL";
+    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    # fields doesn't return zero when it succeeds
-    #unless ($success) {
-    #    print STDERR @$stderr_buf;
-    #}
+
+    # note fields doesn't return zero when it succeeds.
     $fields_output = join "", @$stdout_buf;
 }
-my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
+my (undef, $extname, $extver, $req_name, $action, $email) = split " ", $fields_output;
 
 # make sure the file contains what we are expecting
-# This program shouldn't have been run if the request file is bogus.
-# No need to notify the client
-my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
-my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
-my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
-
+# pstamp_parser_run.pl would not have run this program unless the request file was ok
+my_die("$request_file_name does not contain EXTNAME\n", $PS_EXIT_PROG_ERROR) if !$extname;
+my_die("$request_file_name is not a PS1_PS_REQUEST\n", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
+my_die("REQ_NAME not found in $request_file_name\n", $PS_EXIT_PROG_ERROR)  if (!$req_name);
+my_die("wrong EXTVER $extver found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
+
+if ($extver >= 2) {
+    # We have a version 2 file. Require that the new keywords be supplied. 
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    # XXX check for "valid" $email
+} else {
+    # for version 1 file the action is process and email is not used
+    $action = 'PROCESS';
+    $email = 'null';
+}
+
+print "Request Header Keywords EXTVER: $extver REQ_NAME: $req_name ACTION: $action EMAIL: $email\n";
 
 # check for duplicate request name
@@ -161 +179 @@
     # the output data store's product name
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
+    $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
@@ -174 +193 @@
 
 #
-# now convert the request table to an array of metadata config docs
+# now convert the request table to an array of metadatas
 #
 
@@ -191 +210 @@
     my $dtime_request_file = gettimeofday() - $start_request_file;
     print "Time to read and parse request file: $dtime_request_file\n";
-
-}
-
-#
-# Loop over rows in the request file collecting consecutive rows that have the "same images of interest"
-# in the sense that their selection parameters will yield the same "Runs".
-# Process the groups of rows together to reduce lookup time and to potentially make multiple
-# stamps from the same ppstamp process.
-#
-my @rowList;
+}
+
+my $nRows = scalar @$rows;
+print "\n$nRows rows read from request file\n";
+
+
 my $num_jobs = 0;
 my $imageList;
 my $stage;
-my $need_magic;
 foreach my $row (@$rows) {
-    # santiy check the paramaters
+
+    if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
+        # this is a big request and it came from the upload page and doesn't have a specific label assigned
+        # change it to the generic one that has lower with lower priority
+        $label = 'WEB.BIG';
+        print "\nChanging label for big WEB.UP request to $label\n";
+
+        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
+
+    # validate the paramaters
     if (!checkRow($row)) {
         # when it enconters an error checkRow adds a fake job with an appropriate error code to the database
@@ -216 +245 @@
     $row->{error_code} = 0;
 
-    if (scalar @rowList == 0) {
-        push @rowList, $row;
-        next;
-    }
-
-    my $firstRow = $rowList[0];
-    if (same_images_of_interest($firstRow, $row)) {
-        # add this row to the list and move on
-        push @rowList, $row;
-        next;
-    }
-
-    # the images of interest for this new row doesn't match the list. 
-    # process the list ...
-    $num_jobs += processRows($action, \@rowList);
-
-    # and reset the list to contain just the new row
-    @rowList = ($row);
-}
-
-# out of rows process the list
-if (scalar @rowList > 0) {
-    $num_jobs += processRows($action, \@rowList);
+    $num_jobs += processRow($action, $row);
 }
 
@@ -252 +259 @@
     my $row = shift;
 
+    # check validity of the paramters in a request specification
     # If we encounter an error for a particular row add a job with the proper fault code.
+    # also adjust some paramterers like filter and set defaults for some others
 
     my $stage = $row->{IMG_TYPE};
@@ -284 +293 @@
     }
 
-
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
@@ -314 +322 @@
         return 0;
     }
+
+    my $fwhm_min = $row->{FWHM_MIN};
+    if (!defined $fwhm_min) {
+        $row->{FWHM_MIN} = 0;
+    } elsif (!validNumber($fwhm_min)) {
+        print STDERR "$fwhm_min is not a valid FWHM_MIN\n";
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    my $fwhm_max = $row->{FWHM_MAX};
+    if (!defined $fwhm_max) {
+        $row->{FWHM_MAX} = 0;
+    } elsif (!validNumber($fwhm_max)) {
+        print STDERR "$fwhm_max is not a valid FWHM_MAX\n";
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
     my $data_group = $row->{DATA_GROUP};
     if (!defined $data_group) {
@@ -363 +389 @@
     }
 
-    # $mode list_uri is a debugging mode (it may used by the http interface)
-    # if this happens just croak
-   # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
-
-
     if ($req_type eq "bycoord") {
         if (!$skycenter) {
@@ -384 +405 @@
     }
 
+
     return 1;
 }
@@ -392 +414 @@
 
     $num_jobs = 1;
+    return $num_jobs;
+}
+
+sub processRow {
+    my $action  = shift;
+    my $row     = shift;
+
+    my $num_jobs = 0;
+
+    my $project  = $row->{PROJECT};
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+    if (!$proj_hash) {
+        insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PROJECT);
+        $num_jobs++;
+        return $num_jobs;
+    }
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+
+    my $req_type    = $row->{REQ_TYPE};
+    my $rownum      = $row->{ROWNUM};
+
+    # Since user can get unmagicked data "by coordinate" requests can go back in time
+    # to dredge unusable data from the "dark days"...
+    if ($req_type eq 'bycoord' and $row->{IMG_TYPE} ne 'stack' and $row->{MJD_MIN} == 0) {
+        # ... so unless the user sets mjd_min clamp it to 2009-04-01
+        # XXX: This value should live in the pstampProject table not be hardcoded here
+        $row->{MJD_MIN} = 54922;
+    }
+
+    if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) {
+        # avoid error from print below if $id isn't needed
+        $row->{ID} = "" if !$row->{ID};
+    }
+    
+    # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
+    # request specification. An array reference is returned.
+    my $start_locate = gettimeofday();
+
+    print "\nCalling new_locate_images for row: $rownum\n";
+
+    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+
+    my $dtime_locate = gettimeofday() - $start_locate;
+    print "Time to locate_images for row $rownum $dtime_locate\n";
+
+    my @rowList = ($row);
+    $num_jobs += queueJobs($action, \@rowList, $imageList);
+
+    # if this row slipped through without a job being added add one. 
+    if ($row->{job_num} == 0) {
+        print "row $row->{ROWNUM} produced no jobs\n";
+        print STDERR "row $row->{ROWNUM} produced no jobs\n";
+        my $error_code = $row->{error_code};
+        $error_code =  $PSTAMP_NO_IMAGE_MATCH if !$error_code;
+        insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
+    }
+
     return $num_jobs;
 }
@@ -401 +484 @@
 
     if ($action eq 'LIST') {
+        # LIST is not allowed by caller. Can't get here.
         return list_targets($rowList);
     }
@@ -420 +504 @@
     }
     my $req_type  = $row->{REQ_TYPE};
-    $stage        = $row->{IMG_TYPE};
+    my $stage     = $row->{IMG_TYPE};
     my $id        = $row->{ID};
     my $component = $row->{COMPONENT};
@@ -436 +520 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    $need_magic    = $proj_hash->{need_magic};
+    my $need_magic    = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -496 +580 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component,
+    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -506 +590 @@
     $row->{need_magic} = $need_magic;
 
-    $num_jobs += queueJobs($mode, \@rowList, $imageList);
+    $num_jobs += queueJobs($action, $rowList, $imageList);
 
     # if a row slipped through with no jobs add a faulted one
-    foreach my $row (@rowList) {
+    foreach my $row (@$rowList) {
         if ($row->{job_num} == 0) {
             print "row $row->{ROWNUM} produced no jobs\n";
@@ -528 +612 @@
     my $image = shift;
     my $need_magic = shift;
-    my $mode = shift;
+    my $action = shift;
 
     my $rownum = $row->{ROWNUM};
@@ -611 +695 @@
     write_params($output_base, $image);
 
-    my $newState = "run";
+    my $newState = $action eq 'PROCESS' ? "run" : "parsed";
     my $fault = 0;
     my $dep_id;
 
-    queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
+    queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
 
     my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
@@ -644 +728 @@
 sub queueJobs
 {
-    my $mode = shift;
+    my $action = shift;
     my $rowList = shift;
     my $imageList = shift;
 
-    my $firstRow = $rowList[0];
+    my $firstRow = $rowList->[0];
     my $stage    = $firstRow->{IMG_TYPE};
     my $job_type = $firstRow->{JOB_TYPE};
@@ -656 +740 @@
 
     if ($mode eq "list_uri") {
+        $num_jobs = $imageList ? scalar @$imageList : 0;
+        print "List of $num_jobs Images selected for row: $firstRow->{ROWNUM}\n";
         foreach my $image (@$imageList) {
             print "$image->{image}\n";
+            ++$firstRow->{job_num};
         }
     } elsif ($job_type eq "get_image") {
@@ -664 +751 @@
         my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
 
-        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $action);
 
     } else {
@@ -692 +779 @@
                 my $row = $rowList->[$i];
 
-                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode);
+                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $action);
             }
         }
@@ -706 +793 @@
     my $stage = shift;
     my $need_magic = shift;
-    my $mode = shift;
+    my $action = shift;
 
     my $num_jobs = 0;
@@ -753 +840 @@
         write_params($output_base, $image);
 
-        my $newState = "run";
+        my $newState = $action eq 'PROCESS' ? "run" : "parsed";
         my $fault = 0;
         my $dep_id;
 
-        queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
+        queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
 
         $num_jobs++;
@@ -899 +986 @@
 sub get_dependent 
 {
-    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
+    my ($action, $r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
 
     # chipRun's can be in full state if destreaking is necessary
@@ -918 +1005 @@
     my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir";
     $command .= " -need_magic" if $need_magic;
-
-    # compute rlabel for the run.
-    # XXX: This bit of policy shouldn't be buried so deeply in the code
-    # For now use one that implies 'postage stamp server' 'update' 'request_label"
-    my $rlabel = "ps_ud_" . $label if $label;
-    $command .= " -rlabel $rlabel" if $rlabel;
+    $command .= ' -hold' if $action eq 'PREVIEW';
+
+    if ($label) {
+        # compute rlabel for the run.
+        # XXX: This bit of policy shouldn't be buried so deeply in the code
+        # For now use one that implies 'postage stamp server' 'update' 'request_label"
+        my $rlabel = "ps_ud_" . $label;
+        $command .= " -rlabel $rlabel";
+    }
 
     if (!$no_update) {
@@ -931 +1021 @@
             my $fault = $error_code >> 8;
             print STDERR "$command failed with fault $fault\n";
-            if ($fault < 10) {
+            if ($fault < $PSTAMP_FIRST_ERROR_CODE) {
                 # pstamptool returns an error if an existing depenent is faulted
-                # Set the object to not available even if the fault < 10
+                # Set the object to not available even if the fault < $PSTAMP_FIRST_ERROR_CODE
                 # which is nominally a recoverable error in order to keep
                 # the request from faulting (which can be very expensive if
@@ -939 +1029 @@
                 $fault = $PSTAMP_NOT_AVAILABLE
             }
-            if ($fault >= 10) {
+            if ($fault >= $PSTAMP_FIRST_ERROR_CODE) {
                 $$r_dep_id = 0;
                 $$r_fault = $fault;
@@ -967 +1057 @@
 
 sub queueUpdatesIfNeeded {
+    my $action = shift;
     my $stage = shift;
     my $image = shift;
@@ -973 +1064 @@
     my $r_fault = shift;
     my $r_dep_id = shift;
+
+    my $need_magic = 0;
 
     if ($stage ne 'raw') {
@@ -1016 +1109 @@
                     # set up to queue an update run
                     my $require_magic = ($need_magic or $image->{magicked});
-                    get_dependent(\$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
+                    get_dependent($action, \$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
                         $run_state, $stage, $image->{stage_id}, $image->{component}, $require_magic );
                 }
@@ -1070 +1163 @@
     my $fault = shift;
 
-    carp $msg;
+    print STDERR $msg;
 
     # we don't fault the request here pstamp_parser_run.pl handles that if necessary

branches/eam_branches/ipp-20130307/pstamp/src/ppstampParseCamera.c

@@ -95 +95 @@
     }
         
-
-    // XXX: create a filerule for PPSTAMP.ASTROM
     pmFPAfile *astrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM");
     if (!status) {
@@ -103 +101 @@
     }
     if (astrom) {
-        psLogMsg ("ppstamp", 3, "using supplied astrometry\n");
+        psLogMsg ("ppstamp", 3, "Using supplied astrometry\n");
     } else {
-        psLogMsg ("ppstamp", 3, "using header astrometry\n");
+        psLogMsg ("ppstamp", 3, "Using header astrometry\n");
     }
-
-#ifdef notyet
-    // add recipe options supplied on command line
-    psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, RECIPE_NAME);
-#endif
 
     // Set up the output target
@@ -138 +131 @@
             }
         } else {
-            psLogMsg ("ppstamp", PS_LOG_INFO, "output sources file requested but no input supplied. Will try the supplied astrometry file.\n");
+            psLogMsg ("ppstamp", PS_LOG_INFO, "Output sources file requested but no -sources supplied.\n");
+            psLogMsg ("ppstamp", PS_LOG_INFO, "Will use the sources in the supplied astrometry file.\n");
         }
         

branches/eam_branches/ipp-20130307/tools/heathers_plots/heathers_plots.py

@@ -1 @@
-#!/usr/bin env python
+#!/usr/local/bin/python2.7
 
 """
@@ -10 +10 @@
 import numpy as np
 import matplotlib as mpl
+mpl.use('Agg')
 import matplotlib.pyplot as plt
 import pylab
@@ -24 +25 @@
     IMAGES_DIRECTORY = "/data/ipp009.0/czartool_plots/heathers_plots"
     CAM_WEB_FILENAME = "/data/ipp009.0/czartool_plots/heathers_plots/cam.png"
+    CAM_MERGED_WEB_FILENAME = "/data/ipp009.0/czartool_plots/heathers_plots/cam_merged.png"
     SKYCAL_WEB_FILENAME = "/data/ipp009.0/czartool_plots/heathers_plots/skycal.png"
+    SKYCAL_MERGED_WEB_FILENAME = "/data/ipp009.0/czartool_plots/heathers_plots/skycal_merged.png"
 
     def __init__(self):
@@ -37 +40 @@
 WHERE stage = 'cam' and addRun.dvodb = 'LAP.20120706'
 """
+
+    CAM_MERGED_STATEMENT = """SELECT ra, decl, mergedvodbRun.state, mergedvodbProcessed.fault 
+FROM mergedvodbRun 
+ LEFT JOIN mergedvodbProcessed USING(merge_id)
+ LEFT JOIN minidvodbRun USING(minidvodb_id)
+ LEFT JOIN addRun USING(minidvodb_name)
+ LEFT JOIN addProcessedExp USING(add_id)
+ JOIN camRun ON stage_id = cam_id
+ JOIN chipRun USING(chip_id)
+ JOIN rawExp USING(exp_id)
+WHERE stage = 'cam' AND mergedvodbRun.mergedvodb = 'LAP.20120706'
+"""
+
     SKYCAL_STATEMENT = """SELECT radeg*3.1415927/180., decdeg*3.1415927/180., addRun.state, addProcessedExp.fault 
 FROM addRun
@@ -48 +64 @@
 WHERE stage = 'skycal' AND addRun.dvodb = 'LAP.20120706' 
 """
+
+    SKYCAL_MERGED_STATEMENT = """SELECT radeg*3.1415927/180., decdeg*3.1415927/180., mergedvodbRun.state, mergedvodbProcessed.fault
+FROM mergedvodbRun
+ LEFT JOIN mergedvodbProcessed USING(merge_id)
+ LEFT JOIN minidvodbRun USING(minidvodb_id)
+ LEFT JOIN addRun USING(minidvodb_name)
+ LEFT JOIN addProcessedExp USING(add_id)
+ JOIN skycalRun ON skycal_id = stage_id
+ JOIN skycalResult USING(skycal_id)
+ JOIN staticskyInput USING(sky_id, stack_id)
+ JOIN staticskyRun USING (sky_id)
+ JOIN stackRun using (stack_id)
+ JOIN skycell using (tess_id, skycell_id)
+WHERE stage = 'skycal' AND addRun.dvodb = 'LAP.20120706'"""
 
     def fetch_data(self, statement):
@@ -95 +125 @@
         try:
             os.unlink(web_filename)
-        except OSError: # No such file or directory
-            pass
+        except OSError, e: # No such file or directory
+            print e
         os.symlink(new_filename, web_filename)
 
@@ -115 +145 @@
         self.relink_webfiles(HeathersPlot.CAM_WEB_FILENAME, new_filename)
 
+    def update_cam_merged_plot(self):
+        HeathersPlot.mkdir(HeathersPlot.IMAGES_DIRECTORY)
+        (ras, decs, states) = self.fetch_data(HeathersPlot.CAM_MERGED_STATEMENT)
+        new_filename = self.plot(ras, decs, states, 'Camera_Merged')
+        self.relink_webfiles(HeathersPlot.CAM_MERGED_WEB_FILENAME, new_filename)
+
     def update_skycal_plot(self):
         HeathersPlot.mkdir(HeathersPlot.IMAGES_DIRECTORY)
@@ -121 +157 @@
         self.relink_webfiles(HeathersPlot.SKYCAL_WEB_FILENAME, new_filename)
 
+    def update_skycal_merged_plot(self):
+        HeathersPlot.mkdir(HeathersPlot.IMAGES_DIRECTORY)
+        (ras, decs, states) = self.fetch_data(HeathersPlot.SKYCAL_MERGED_STATEMENT)
+        new_filename = self.plot(ras, decs, states, 'SkyCal_Merged')
+        self.relink_webfiles(HeathersPlot.SKYCAL_MERGED_WEB_FILENAME, new_filename)
+
 if __name__ == "__main__":
     hp = HeathersPlot()
     hp.update_cam_plot()
     hp.update_skycal_plot()
+    hp.update_cam_merged_plot()
+    hp.update_skycal_merged_plot()