Changeset 36627 for branches/eam_branches/ipp-20140206

Timestamp:

Apr 1, 2014, 11:03:38 AM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20140206

Files:

• . (modified) (1 prop)
• ippScripts/scripts/dist_advancerun.pl (modified) (5 diffs)
• ippScripts/scripts/dist_bundle.pl (modified) (3 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/sc_prepare_chip.pl (copied) (copied from trunk/ippScripts/scripts/sc_prepare_chip.pl )
• ippScripts/scripts/sc_remote_exec.pl (copied) (copied from trunk/ippScripts/scripts/sc_remote_exec.pl )
• ippScripts/scripts/sc_validate_files.pl (copied) (copied from trunk/ippScripts/scripts/sc_validate_files.pl )
• ippScripts/scripts/sc_validate_processing.pl (copied) (copied from trunk/ippScripts/scripts/sc_validate_processing.pl )
• ippTasks/staticsky.pro (modified) (2 diffs)
• ippToPsps/jython/cleanup.py (modified) (2 diffs)
• ippToPsps/jython/dxlayer.py (modified) (4 diffs)
• ippToPsps/jython/odm.py (modified) (2 diffs)
• ippToPsps/perl/deleteFromDXLayer.pl (modified) (1 diff)
• ippToPsps/perl/deleteFromDXLayer2.pl (modified) (1 diff)
• ippTools/share/Makefile.am (modified) (5 diffs)
• ippTools/share/detselect_show.sql (copied) (copied from trunk/ippTools/share/detselect_show.sql )
• ippTools/share/disttool_definebyquery_ff.sql (copied) (copied from trunk/ippTools/share/disttool_definebyquery_ff.sql )
• ippTools/share/disttool_pending_ff.sql (copied) (copied from trunk/ippTools/share/disttool_pending_ff.sql )
• ippTools/share/disttool_toadvance.sql (modified) (1 diff)
• ippTools/share/fftool_export_input.sql (copied) (copied from trunk/ippTools/share/fftool_export_input.sql )
• ippTools/share/fftool_export_result.sql (copied) (copied from trunk/ippTools/share/fftool_export_result.sql )
• ippTools/share/fftool_export_run.sql (copied) (copied from trunk/ippTools/share/fftool_export_run.sql )
• ippTools/share/fftool_export_summary.sql (copied) (copied from trunk/ippTools/share/fftool_export_summary.sql )
• ippTools/share/fftool_result.sql (modified) (1 diff)
• ippTools/share/fftool_summary.sql (modified) (1 diff)
• ippTools/src (modified) (1 prop)
• ippTools/src/detselect.c (modified) (3 diffs)
• ippTools/src/detselect.h (modified) (1 diff)
• ippTools/src/detselectConfig.c (modified) (2 diffs)
• ippTools/src/difftool.c (modified) (2 diffs)
• ippTools/src/disttool.c (modified) (1 diff)
• ippTools/src/fftool.c (modified) (8 diffs)
• ippTools/src/fftool.h (modified) (1 diff)
• ippTools/src/fftoolConfig.c (modified) (2 diffs)
• ippTools/src/laptool.c (modified) (2 diffs)
• ippTools/src/remotetool.c (modified) (3 diffs)
• ippconfig/recipes/filerules-split.mdc (modified) (2 diffs)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)
• ppBackground/src/Makefile.am (modified) (3 diffs)
• ppBackground/src/ppBackground.c (modified) (1 diff)
• ppBackground/src/ppBackgroundStack.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStack.c )
• ppBackground/src/ppBackgroundStack.h (copied) (copied from trunk/ppBackground/src/ppBackgroundStack.h )
• ppBackground/src/ppBackgroundStackArguments.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackArguments.c )
• ppBackground/src/ppBackgroundStackCamera.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackCamera.c )
• ppBackground/src/ppBackgroundStackData.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackData.c )
• ppBackground/src/ppBackgroundStackLoop.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackLoop.c )
• ppBackground/src/ppBackgroundStackMath.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackMath.c )
• ppStack/src/ppStackReadout.c (modified) (2 diffs)
• ppSub/src/ppSubMatchPSFs.c (modified) (1 diff)
• psLib/src/fits/psFitsImage.c (modified) (1 diff)
• psphot/src (modified) (1 prop)
• psphot/src/Makefile.am (modified) (1 diff)
• psphot/src/psphot.h (modified) (1 diff)
• psphot/src/psphotApResid.c (modified) (2 diffs)
• psphot/src/psphotChoosePSF.c (modified) (1 diff)
• psphot/src/psphotCleanup.c (modified) (1 diff)
• psphot/src/psphotLensing.c (copied) (copied from trunk/psphot/src/psphotLensing.c )
• psphot/src/psphotReadout.c (modified) (2 diffs)
• psphot/src/psphotStackImageLoop.c (modified) (1 prop)
• psphot/src/psphotStackMatchPSFsNext.c (modified) (1 diff)
• psphot/src/psphotStackReadout.c (modified) (1 diff)
• pstamp/scripts/pstamp_insert_request.pl (modified) (3 diffs)
• pstamp/scripts/pstampparse.pl (modified) (5 diffs)
• tools/dlapstacks (modified) (1 diff)
• tools/fixburntool (modified) (1 diff)

branches/eam_branches/ipp-20140206/ippScripts/scripts/dist_advancerun.pl

@@ -69 +69 @@
 my $stacktool   = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1);
 my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
 if ($missing_tools) {
@@ -77 +78 @@
 
 my $tool_cmd;
+my $tool_cmd2;
 my $list_mode;
 my $component_key;
@@ -127 +129 @@
     $list_mode = "-diffskyfile";
     $component_key = "skycell_id";
+} elsif ($stage eq "ff") {
+    $tool_cmd = "$fftool -ff_id";
+    $list_mode = "-result";
+    $component_key = "dist_component";
+    $tool_cmd2 = "$fftool -summary -ff_id"
 } else {
     &my_die("Unexpected stage: $stage", $dist_id, $PS_EXIT_CONFIG_ERROR);
@@ -132 +139 @@
 
 $tool_cmd .= " $stage_id";
+$tool_cmd2 .= " $stage_id" if $tool_cmd2;
 
 my $exportarg = '-exportrun';
@@ -215 +223 @@
     my $components = parse_md_list($metadata) or
         &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    if ($tool_cmd2) {
+        my $command = "$tool_cmd2";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+            unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+        }
+        if (@$stdout_buf == 0) {
+            &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+        }
+        my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+        my $more_components = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+        if (scalar @$more_components) {
+            push @$components, @$more_components;
+        }
+    }
 
     open MANIFEST, ">$resolved" or

branches/eam_branches/ipp-20140206/ippScripts/scripts/dist_bundle.pl

@@ -496 +496 @@
     } elsif ($stage eq "stack") {
         $type = $stack_cleaned{$rule};
-    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary') {
+    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary' or $stage eq 'ff') {
         $type = $empty_cleaned{$rule};
     } else {
@@ -588 +588 @@
         # stack_summary stage does not use a config dump file.
         return build_stack_summary_file_list($path_base, \@file_list);
+    } elsif ($stage eq "ff") {
+        if ($component eq 'summary') {
+            # full force summary does not use a config dump file. Cobble together a file list
+            return build_ff_summary_file_list($path_base, \@file_list);
+        } else {
+            $config_file_rule = "PSPHOT.SKY.CONFIG";
+        }
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -742 +749 @@
 }
 
+# psphotFullForceSummary does not produce a config dump file cobble together a file list
+sub build_ff_summary_file_list {
+    my ($path_base, $file_list) = @_;
+
+    my %cmf ;
+    $cmf{file_rule} = "SOURCES";
+    $cmf{name} = "$path_base.cmf";
+    push @$file_list, \%cmf;
+
+    my %log;
+    $log{file_rule} = "LOG";
+    $log{name} = "$path_base.log";
+    push @$file_list, \%log;
+
+    return $file_list;
+}
+
 sub my_die
 {

branches/eam_branches/ipp-20140206/ippTasks/staticsky.pro

@@ -64 +64 @@
 end
 
+$RA_POLL_MAX = 0
+
+macro set.ra.max
+    if ($0 != 2) 
+        echo "USAGE: set.ra.max (ra_max_deg)"
+        break
+    end
+    $RA_POLL_MAX = $1
+end
+macro get.ra.max
+    echo $RA_POLL_MAX
+end
+
 ### Load tasks for staticsky
 ### Tasks are loaded into staticskyResult.
@@ -80 +93 @@
     if ($LABEL:n == 0) break
     $run = staticskytool -todo
+    if ($RA_POLL_MAX > 0) 
+        $run = $run -ra_max $RA_POLL_MAX
+    end
     if ($DB:n == 0)
       option DEFAULT

branches/eam_branches/ipp-20140206/ippToPsps/jython/cleanup.py

@@ -87 +87 @@
         self.logger.infoPair("To delete from DXLayer", "%d" % len(deleteFromDxLayerIDs))
         self.logger.infoPair("To delete from local disk", "%d" % len(deleteFromLocalIDs))
-    
+    #    self.logger.infoPair("Did I get here", "yes?")
         # delete stuff from local disk
         if self.skychunk.deleteLocal:
@@ -110 +110 @@
     
             self.reportResults("datastore", deleteFromDatastoreIDs, count) 
-        
+       # self.logger.infoPair("here","here")
         # remove stuff from DXLayer
         if self.skychunk.deleteDxLayer:
+#            self.logger.infoPair ("skip remove from dxlayer", "HAF")
             count = 0
             for id in deleteFromDxLayerIDs:
-        
+                self.logger.infoPair("delete from dxlayer", id);
                 if self.dxlayer.deleteBatch(id):
+                    self.logger.infoPair("delete dxlayer",id);
                     self.ippToPspsDb.updateDeletedDxlayer(id, 1)
                     count = count + 1

branches/eam_branches/ipp-20140206/ippToPsps/jython/dxlayer.py

@@ -37 +37 @@
        
         if p.returncode != 0:
-           self.logger.debugPair("DXLayer webservice", "failed")
+           self.logger.infoPair("DXLayer webservice", "failed")
            return 0
         else:
-           self.logger.debugPair("DXLayer webservice", "successful")
+           self.logger.infoPair("DXLayer webservice", "successful")
 
         ret = self.decodeDXLayerResponse(tempFile.name)
@@ -48 +48 @@
 
         if ret: self.logger.infoPair("Deleted batch from DXLayer 1", batch) 
-        else: self.logger.errorPair("Unable to delete batch from DXLayer 1" , batch)
+        else: self.logger.infoPair("Unable to delete batch from DXLayer 1" , batch)
 
 
@@ -61 +61 @@
        
         if p.returncode != 0:
-           self.logger.debugPair("DXLayer webservice", "failed")
+           self.logger.infoPair("DXLayer webservice", "failed")
            return 0
         else:
@@ -72 +72 @@
 
         if ret: self.logger.infoPair("Deleted batch from DXLayer 2", batch) 
-        else: self.logger.errorPair("Unable to delete batch from DXLayer 2" , batch)
+        else: self.logger.infoPair("Unable to delete batch from DXLayer 2" , batch)
 
 

branches/eam_branches/ipp-20140206/ippToPsps/jython/odm.py

@@ -78 +78 @@
                 DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text
                 results['DETAILS'] = DETAILS
-            elif re.search("processing", STATE): 
+            elif re.search("processing", STATE):
                 loadedToODM = 1
             if re.search("MergeWorthy", MESSAGE): 
@@ -87 +87 @@
                 mergeWorthy = 1
                 merged = 1
+#            if re.search("released", STATE):
+#                loadedToODM = 1
+#                mergeWorthy = 1
+#                merged = 1
+#            if re.search("released", MESSAGE):
+#                loadedToODM = 1
+#                mergeWorthy = 1
+#                merged = 1
         except:
             pass

branches/eam_branches/ipp-20140206/ippToPsps/perl/deleteFromDXLayer.pl

@@ -18 +18 @@
         );
 
-my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
+#my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
+
+#my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxwebservice.cgi";
+my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxlayer/dxwebservice.cgi";
+
+
 #my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
 

branches/eam_branches/ipp-20140206/ippToPsps/perl/deleteFromDXLayer2.pl

@@ -18 +18 @@
         );
 
-my $dxWebServiceUrl = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
+#my $dxWebServiceUrl = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
+
+#my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb02/cgi-bin/dxwebservice.cgi";
+my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb02/cgi-bin/dxlayer/dxwebservice.cgi";
 #my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
 

branches/eam_branches/ipp-20140206/ippTools/share/Makefile.am

@@ -105 +105 @@
         detselect_search.sql \
         detselect_select.sql \
+        detselect_show.sql \
         dettool_addprocessedexp.sql \
         dettool_childlessrun.sql \
@@ -189 +190 @@
         disttool_definebyquery_diff.sql \
         disttool_definebyquery_fake.sql \
+        disttool_definebyquery_ff.sql \
         disttool_definebyquery_raw.sql \
         disttool_definebyquery_raw_no_magic.sql \
@@ -206 +208 @@
         disttool_pending_diff.sql \
         disttool_pending_fake.sql \
+        disttool_pending_ff.sql \
         disttool_pending_raw.sql \
         disttool_pending_sky.sql \
@@ -497 +500 @@
         fftool_todo.sql \
         fftool_toadvance.sql \
+        fftool_export_run.sql \
+        fftool_export_input.sql \
+        fftool_export_result.sql \
+        fftool_export_summary.sql \
         remotetool_definebyquery_chip.sql \
         remotetool_definebyquery_cam.sql \
@@ -502 +509 @@
         remotetool_definebyquery_stack.sql \
         remotetool_updatepoll.sql
-
-
-

branches/eam_branches/ipp-20140206/ippTools/share/disttool_toadvance.sql

@@ -308 +308 @@
         COUNT(diffSkyfile.skycell_id) = COUNT(distComponent.component)
         AND SUM(distComponent.fault) = 0
+UNION
+-- ff stage
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    distRun.label,
+    clean
+    -- ,COUNT(warp_id)
+    -- ,COUNT(ff_id)
+    FROM distRun
+    JOIN fullForceRun ON stage_id = ff_id
+    JOIN fullForceResult USING(ff_id)
+    JOIN skycalRun USING(skycal_id)
+    JOIN stackRun USING(stack_id)
+    LEFT JOIN distComponent
+        ON distRun.dist_id = distComponent.dist_id
+        AND distComponent.component = CONCAT_WS('.', warp_id, stackRun.skycell_id) 
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distRun.stage = 'ff'
+    GROUP BY
+        distRun.dist_id,
+        ff_id
+    HAVING
+        -- number of dist components is the number of warps plus 1 (for the summary component)
+        COUNT(fullForceResult.ff_id) = COUNT(distComponent.component)
+        AND SUM(distComponent.fault) = 0
 ) as Foo

branches/eam_branches/ipp-20140206/ippTools/share/fftool_result.sql

@@ -1 +1 @@
 SELECT
     fullForceResult.*,
-    fullForceRun.skycal_id,
+    fullForceRun.state,
     fullForceRun.label,
     fullForceRun.data_group,
+    fullForceRun.skycal_id,
+    fullForceRun.workdir,
+    CONCAT_WS('.', warp_id, skycell_id) AS dist_component,
     stackRun.tess_id,
     stackRun.skycell_id,

branches/eam_branches/ipp-20140206/ippTools/share/fftool_summary.sql

@@ -5 +5 @@
     fullForceRun.data_group,
     fullForceRun.skycal_id,
+    fullForceRun.workdir,
+    'summary' AS dist_component,
     stackRun.tess_id,
     stackRun.skycell_id,

branches/eam_branches/ipp-20140206/ippTools/src

@@ -48 +48 @@
 releasetool
 fftool
+remotetool
+

@@ -48 +48 @@
 releasetool
 fftool
+remotetool
+

branches/eam_branches/ipp-20140206/ippTools/src/detselect.c

@@ -29 +29 @@
 static bool searchMode(pxConfig *config);
 static bool selectMode(pxConfig *config);
+static bool showMode(pxConfig *config);
+
 
 # define MODECASE(caseName, func) \
@@ -50 +52 @@
         MODECASE(DETSELECT_MODE_SEARCH,        searchMode);
         MODECASE(DETSELECT_MODE_SELECT,        selectMode);
+        MODECASE(DETSELECT_MODE_SHOW,          showMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -241 +244 @@
     return true;
 }
+
+static bool showMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_STR(config->args, where, "-inst",      "camera", "==");
+    PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "==");
+    PXOPT_COPY_STR(config->args, where, "-det_type",  "det_type", "==");
+    PXOPT_COPY_STR(config->args, where, "-type",      "det_type", "==");
+
+    psString query = pxDataGet("detselect_show.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // we choose the single detrend image which matches all criteria and has
+    // the latest insertion date
+
+    psStringAppend(&query, " ORDER BY registered DESC, iteration DESC");
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        // XXX check psError here
+        psError(PS_ERR_UNKNOWN, false, "no detRun rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+    return(true);
+}

branches/eam_branches/ipp-20140206/ippTools/src/detselect.h

@@ -27 +27 @@
     DETSELECT_MODE_SEARCH,
     DETSELECT_MODE_SELECT,
+    DETSELECT_MODE_SHOW
 } detselectMode;
 

branches/eam_branches/ipp-20140206/ippTools/src/detselectConfig.c

@@ -65 +65 @@
     psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-class_id", 0,                     "search by class ID", NULL);
     psMetadataAddBool(selectArgs, PS_LIST_TAIL, "-simple",  0,                      "use the simple output format", false);
+
+    // -show
+    psMetadata *showArgs = psMetadataAlloc();
+    psMetadataAddStr(showArgs, PS_LIST_TAIL, "-inst",       0,       "search by camera", NULL);
+    psMetadataAddStr(showArgs, PS_LIST_TAIL, "-telescope",  0,       "search by telescope", NULL);
+    psMetadataAddStr(showArgs, PS_LIST_TAIL, "-det_type",   0,       "search by detrend type", NULL);
+    psMetadataAddStr(showArgs, PS_LIST_TAIL, "-type",       0,       "search by detrend type (alias for -det_type)", NULL);
+    psMetadataAddBool(showArgs, PS_LIST_TAIL, "-simple",     0,       "use the simple output format", false);
     
     psMetadata *argSets = psMetadataAlloc();
@@ -71 +79 @@
     PXOPT_ADD_MODE("-search",  "search for an appropriate detrend", DETSELECT_MODE_SEARCH, searchArgs);
     PXOPT_ADD_MODE("-select",  "retreive detrend information", DETSELECT_MODE_SELECT, selectArgs);
+    PXOPT_ADD_MODE("-show",    "show all active detrends of this type with constraints", DETSELECT_MODE_SHOW, showArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/eam_branches/ipp-20140206/ippTools/src/difftool.c

@@ -1349 +1349 @@
     if (!mdok) {
         psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psString warpDataGroup = psMetadataLookupStr(&mdok, row, "warpDataGroup");
+    if (!mdok) {
+        psError(PXTOOLS_ERR_PROG, false, "warpDataGroup not found");
         if (!psDBRollback(config->dbh)) {
           psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1391 +1399 @@
                                         workdir,
                                         label,
-                                        data_group ? data_group : label,
+                                        data_group ? data_group : warpDataGroup,
                                         dist_group,
                                         reduction,

branches/eam_branches/ipp-20140206/ippTools/src/disttool.c

@@ -413 +413 @@
 
       magic = false;
+    } else if (!strcmp(stage, "ff")) {
+      runJoinStr = "fullForceRun.ff_id";
+      query = pxDataGet("disttool_definebyquery_ff.sql");
+      if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+        psFree(where);
+        return(false);
+      }
+
+      if (label) {
+        psStringAppend(&query, " AND (fullForceRun.label = '%s') ", label);
+      }
+      if (dist_group) {
+        psStringAppend(&query, " AND (fullForceRun.dist_group = '%s') ", dist_group);
+      }
+
+      magic = false;
     } else {
         psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage);

branches/eam_branches/ipp-20140206/ippTools/src/fftool.c

@@ -44 +44 @@
 static bool updatesummaryMode(pxConfig *config);
 static bool summaryMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
+static bool summaryMode(pxConfig *config);
 
 static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state);
@@ -77 +80 @@
         MODECASE(FFTOOL_MODE_UPDATESUMMARY,     updatesummaryMode);
         MODECASE(FFTOOL_MODE_SUMMARY,           summaryMode);
+        MODECASE(FFTOOL_MODE_EXPORTRUN,         exportrunMode);
+        MODECASE(FFTOOL_MODE_IMPORTRUN,         importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -134 +139 @@
 
     psMetadata *warpWhereMD = psMetadataAlloc();
-    pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label",     "warpRun.label",           "LIKE");
-    pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","warpRun.data_group",      "LIKE");
-    PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id",        "warpRun.warp_id",         "==");
+    pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label",     "fullForceRun.label",           "LIKE");
+    pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","fullForceRun.data_group",      "LIKE");
+    PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id",        "fullForceRun.warp_id",         "==");
     if (!psListLength(warpWhereMD->list)) {
         psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required");
@@ -150 +155 @@
     }
 
-    PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id",       "warpRun.tess_id",         "==");
-    pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter",        "warpRun.filter",          "LIKE");
+    PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id",       "fullForceRun.tess_id",         "==");
+    pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter",        "fullForceRun.filter",          "LIKE");
     if (!pxskycellAddWhere(config, warpWhereMD)) {
         psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
@@ -175 +180 @@
     psString whereClause = psDBGenerateWhereConditionSQL(skycalWhereMD, NULL);
     psStringAppend(&where, "\nAND %s", whereClause);
-    psStringAppend(&select, where);
+    psStringAppend(&select, " %s", where);
     psFree(whereClause);
     psFree(skycalWhereMD);
@@ -504 +509 @@
     PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackRun.tess_id", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filter",      "stackRun.filter", "LIKE");
     PXOPT_COPY_S16(config->args, where, "-fault",      "staticskyResult.fault", "==");
     pxskycellAddWhere(config, where);
@@ -864 +870 @@
     PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackRun.tess_id", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filter",     "stackRun.filter", "LIKE");
     PXOPT_COPY_S16(config->args, where, "-fault",      "staticskyResult.fault", "==");
     pxskycellAddWhere(config, where);
@@ -933 +940 @@
     return true;
 }
+
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    // PXOPT_LOOKUP_S64(det_id,  config->args, "-warp_id", true,  false);
+    PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+    PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+//    PXOPT_LOOKUP_BOOL(clean,  config->args, "-clean", false);
+
+    FILE *f = fopen (outfile, "w");
+    if (f == NULL) {
+        psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+        return false;
+    }
+
+    if (!pxExportVersion(config, f)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file");
+        return false;
+    }
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "==");
+
+    ExportTable tables [] = {
+      {"fullForceRun", "fftool_export_run.sql"},
+      {"fullForceInput", "fftool_export_input.sql"},
+      {"fullForceResult", "fftool_export_result.sql"},
+      {"fullForceSummary", "fftool_export_summary.sql"},
+    };
+
+    int numTables = sizeof(tables)/sizeof(tables[0]);
+
+    for (int i=0; i < numTables; i++) {
+      psString query = pxDataGet(tables[i].sqlFilename);
+      if (!query) {
+          psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+          return false;
+      }
+
+      if (where && psListLength(where->list)) {
+          psString whereClause = psDBGenerateWhereSQL(where, NULL);
+          psStringAppend(&query, " %s", whereClause);
+          psFree(whereClause);
+      }
+
+      // treat limit == 0 as "no limit"
+      if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+      }
+
+      if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+      }
+      psFree(query);
+
+      psArray *output = p_psDBFetchResult(config->dbh);
+      if (!output) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+          return false;
+      }
+      if (!psArrayLength(output)) {
+        psError(PS_ERR_UNKNOWN, true, "no rows found");
+        psFree(output);
+        return false;
+      }
+
+      // we must write the export table in non-simple (true) format
+      if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+      }
+      psFree(output);
+    }
+
+    fclose (f);
+
+    return true;
+}
+
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+
+  int numImportTables = 3;
+
+  char tables[3] [80] = {"fullForceInput", "fullForceResult", "fullForceSummary"};
+
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+
+#ifdef notdef
+  fprintf (stderr, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+#endif
+
+  if (!pxCheckImportVersion(config, input)) {
+      psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed");
+      return false;
+  }
+
+  psMetadataItem *item = psMetadataLookup (input, "fullForceRun");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  fullForceRunRow *fullForceRun = fullForceRunObjectFromMetadata (entry->data.md);
+  fullForceRunInsertObject (config->dbh, fullForceRun);
+
+  // fprintf (stdout, "---- warp run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+
+  for (int i = 0; i < numImportTables; i++) {
+    item = psMetadataLookup (input, tables[i]);
+    psAssert (item, "entry not in input?");
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+
+    switch (i) {
+      case 0:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          fullForceInputRow *fullForceInput = fullForceInputObjectFromMetadata (entry->data.md);
+          fullForceInputInsertObject (config->dbh, fullForceInput);
+
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+
+      case 1:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          fullForceResultRow *fullForceResult = fullForceResultObjectFromMetadata (entry->data.md);
+          fullForceResultInsertObject (config->dbh, fullForceResult);
+
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+
+      case 2:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          fullForceSummaryRow *fullForceSummary = fullForceSummaryObjectFromMetadata (entry->data.md);
+          fullForceSummaryInsertObject (config->dbh, fullForceSummary);
+
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+    }
+  }
+  return true;
+}

branches/eam_branches/ipp-20140206/ippTools/src/fftool.h

@@ -36 +36 @@
     FFTOOL_MODE_REVERTSUMMARY,
     FFTOOL_MODE_UPDATESUMMARY,
-    FFTOOL_MODE_SUMMARY
+    FFTOOL_MODE_SUMMARY,
+    FFTOOL_MODE_EXPORTRUN,
+    FFTOOL_MODE_IMPORTRUN,
 } fftoolMode;
 

branches/eam_branches/ipp-20140206/ippTools/src/fftoolConfig.c

@@ -186 +186 @@
     pxskycellAddArguments(revertsummaryArgs);
 
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ff_id", 0,          "export this full force ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+//    psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "export run in cleaned state", false);
+
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
+
+
+
     psFree(now);
 
@@ -203 +216 @@
     PXOPT_ADD_MODE("-revertsummary", "revert faulted summary", FFTOOL_MODE_REVERTSUMMARY,  revertsummaryArgs);
     PXOPT_ADD_MODE("-summary",       "list summary results", FFTOOL_MODE_SUMMARY,  summaryArgs);
+    PXOPT_ADD_MODE("-exportrun",     "list summary results", FFTOOL_MODE_EXPORTRUN,  exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",     "list summary results", FFTOOL_MODE_IMPORTRUN,  importrunArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/eam_branches/ipp-20140206/ippTools/src/laptool.c

@@ -1170 +1170 @@
   // we only want projection cells which do not already of an entry
   psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n");
-  psStringAppend(&query, "\nORDER by projection_cell\n");
+  // Projection cell's don't have a convienient order. Queue runs in the order of lap_id
+  // psStringAppend(&query, "\nORDER by projection_cell\n");
+  psStringAppend(&query, "\nORDER by lap_id_0\n");
 
   if (limit) {
@@ -1289 +1291 @@
     psFree(whereClause);
   }
+  psStringAppend(&query, "\nGROUP by projection_cell ORDER by MIN(lap_id)");
   if (limit) {
     psString limitString = psDBGenerateLimitSQL(limit);

branches/eam_branches/ipp-20140206/ippTools/src/remotetool.c

@@ -149 +149 @@
     psFree(limitString);
   }
-  if (!p_psDBRunQueryF(config->dbh, query)) {
+  if (!p_psDBRunQuery(config->dbh, query)) {
     psError(PS_ERR_UNKNOWN, false, "database error");
     psFree(query);
@@ -313 +313 @@
 {
   PS_ASSERT_PTR_NON_NULL(config, false);
-  PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
+  // PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
   
   psMetadata *where = psMetadataAlloc();
@@ -341 +341 @@
 {
   PS_ASSERT_PTR_NON_NULL(config, false);
-  PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
+  // PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
   
   psMetadata *where = psMetadataAlloc();

branches/eam_branches/ipp-20140206/ippconfig/recipes/filerules-split.mdc

@@ -303 +303 @@
 SKYCELL.TEMPLATE             OUTPUT {OUTPUT}.skycell                  SKYCELL         NONE       FPA        TRUE      NONE
                                                                                                               
-PPSUB.OUTPUT                 OUTPUT {OUTPUT}.fits                     IMAGE           COMP_SUB   FPA        TRUE      NONE
+PPSUB.OUTPUT                 OUTPUT {OUTPUT}.fits                     IMAGE           NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.MASK            OUTPUT {OUTPUT}.mask.fits                MASK            COMP_MASK  FPA        TRUE      NONE
 PPSUB.OUTPUT.VARIANCE        OUTPUT {OUTPUT}.wt.fits                  VARIANCE        COMP_WT    FPA        TRUE      NONE
@@ -311 +311 @@
 PPSUB.OUTPUT.JPEG2           OUTPUT {OUTPUT}.b2.jpg                   JPEG            NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.RESID.JPEG      OUTPUT {OUTPUT}.res.jpg                  JPEG            NONE       FPA        TRUE      NONE
-PPSUB.INVERSE                OUTPUT {OUTPUT}.inv.fits                 IMAGE           COMP_SUB   FPA        TRUE      NONE
+PPSUB.INVERSE                OUTPUT {OUTPUT}.inv.fits                 IMAGE           NONE       FPA        TRUE      NONE
 PPSUB.INVERSE.MASK           OUTPUT {OUTPUT}.inv.mask.fits            MASK            COMP_MASK  FPA        TRUE      NONE
 PPSUB.INVERSE.VARIANCE       OUTPUT {OUTPUT}.inv.wt.fits              VARIANCE        COMP_WT    FPA        TRUE      NONE

branches/eam_branches/ipp-20140206/ppBackground/src/Makefile.am

@@ -1 @@
-bin_PROGRAMS = ppBackground
+bin_PROGRAMS = ppBackground ppBackgroundStack
 
 if HAVE_SVNVERSION
@@ -28 +28 @@
 ppBackground_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSTATS_LIBS)   $(PSPHOT_LIBS)   $(PPBACKGROUND_LIBS)
 
+ppBackgroundStack_CPPFLAGS = $(ppBackground_CPPFLAGS)
+ppBackgroundStack_LDFLAGS = $(ppBackground_LDFLAGS)
+
 ppBackground_SOURCES =          \
         ppBackground.c          \
@@ -38 +41 @@
         ppBackgroundErrorCodes.c        \
         ppBackgroundExit.c
+
+ppBackgroundStack_SOURCES = \
+        ppBackgroundStack.c \
+        ppBackgroundStackArguments.c \
+        ppBackgroundStackCamera.c \
+        ppBackgroundStackData.c \
+        ppBackgroundStackLoop.c \
+        ppBackgroundStackMath.c \
+        ppBackgroundVersion.c \
+        ppBackgroundErrorCodes.c \
+        ppBackgroundExit.c
+
 
 noinst_HEADERS = \

branches/eam_branches/ipp-20140206/ppBackground/src/ppBackground.c

@@ -18 +18 @@
     ppBackgroundErrorRegister();
 
+    printf("Initializing data\n");
     ppBackgroundData *data = ppBackgroundDataInit(&argc, argv);
     if (!data) {
         goto DIE;
     }
-
+    printf("Reading Arguments\n");
     if (!ppBackgroundArguments(data, argc, argv)) {
         goto DIE;
     }
-
+    printf("Setting up Camera\n");
     if (!ppBackgroundCamera(data)) {
         goto DIE;
     }
 
+    printf("Looping over data!\n");
     if (!ppBackgroundLoop(data)) {
         goto DIE;

branches/eam_branches/ipp-20140206/ppStack/src/ppStackReadout.c

@@ -184 +184 @@
     psFree(stack);
 
+    psLogMsg("ppStack", PS_LOG_INFO, "initial stack image sectionNum %d", sectionNum);
+
     sectionNum++;
 
@@ -293 +295 @@
     psFree(stack);
 
+    psLogMsg("ppStack", PS_LOG_INFO, "final stack image sectionNum %d", sectionNum);
+
     sectionNum++;
 

branches/eam_branches/ipp-20140206/ppSub/src/ppSubMatchPSFs.c

@@ -156 +156 @@
     psLogMsg("ppSub", PS_LOG_INFO, "Input FWHM: %f\nReference FWHM: %f\n", inFWHM, refFWHM);
     if (!isfinite(inFWHM) || !isfinite(refFWHM)) {
+#ifdef SET_QUALITY_INSTEAD_OF_ERROR
         psErrorStackPrint(stderr, "Cannot determine FHWM for images, giving up.");
         int error = psErrorCodeLast(); // Error code
         ppSubDataQuality(data, error, PPSUB_FILES_ALL);
         return true;
+#else
+        psError(PPSUB_ERR_DATA, false, "Cannot determine FHWM for images, giving up.");
+        return false;
+#endif
     }
 

branches/eam_branches/ipp-20140206/psLib/src/fits/psFitsImage.c

@@ -753 +753 @@
     }   
 
+    // If there is no options set, remove all our non-standard keywords, because no one asked for them.
+    if (!options) {
+      if (psMetadataLookup(header,"BOFFSET")) {
+        psMetadataRemoveKey(header,"BOFFSET");
+      }
+      if (psMetadataLookup(header,"BSOFTEN")) {
+        psMetadataRemoveKey(header,"BSOFTEN");
+      }
+    }      
+    
     // write the header, if any.
     if (header && !psFitsWriteHeaderImage(fits, header, createPHU)) {

branches/eam_branches/ipp-20140206/psphot/src/Makefile.am

@@ -208 +208 @@
         psphotMakeGrowthCurve.c        \
         psphotMagnitudes.c             \
+        psphotLensing.c        \
         psphotSetMaskBits.c            \
         psphotSkyReplace.c             \

branches/eam_branches/ipp-20140206/psphot/src/psphot.h

@@ -144 +144 @@
 bool            psphotMagnitudesReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf);
 bool            psphotMagnitudes_Threaded (psThreadJob *job);
+
+bool            psphotLensing (pmConfig *config, const pmFPAview *view, const char *filerule);
+bool            psphotLensingReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe);
+bool            psphotLensingPSFtrendsReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe);
 
 bool            psphotEfficiency (pmConfig *config, const pmFPAview *view, const char *filerule);

branches/eam_branches/ipp-20140206/psphot/src/psphotApResid.c

@@ -214 +214 @@
 
         // XXX make this user-configurable?
-        if (source->psfMagErr > 0.01) continue;
+        if (source->psfMagErr > 0.03) continue;
 
         // aperture residual for this source
@@ -267 +267 @@
 
     // set the max order (0 = constant) which the number of psf stars can support:
-    // rule of thumb: require 3 stars per 'cell' (order+1)^2
+    // we require only 3 stars for n = 0, increase stars / cell for higher order
     int MaxOrderForStars = 0;
-    if (Npsf >= 12) MaxOrderForStars = 1; // 4 cells
-    if (Npsf >= 27) MaxOrderForStars = 2; // 9 cells
-    if (Npsf >= 48) MaxOrderForStars = 3; // 16 cells
-    if (Npsf >  75) MaxOrderForStars = 4; // 25 cells
+    if (Npsf >=  16) MaxOrderForStars = 1; // 4 cells
+    if (Npsf >=  54) MaxOrderForStars = 2; // 9 cells
+    if (Npsf >= 128) MaxOrderForStars = 3; // 16 cells
+    if (Npsf >= 300) MaxOrderForStars = 4; // 25 cells
+    if (Npsf >  576) MaxOrderForStars = 5; // 36 cells
 
     pmTrend2DMode mode = PM_TREND_MAP;

branches/eam_branches/ipp-20140206/psphot/src/psphotChoosePSF.c

@@ -399 +399 @@
     char *modelName = pmModelClassGetName (psf->type);
     psLogMsg ("psphot.pspsf", PS_LOG_WARN, "select psf model: %f sec\n", psTimerMark ("psphot.choose.psf"));
-    psLogMsg ("psphot.pspsf", PS_LOG_INFO, "psf model %s, ApResid: %f +/- %f\n", modelName, psf->ApResid, psf->dApResid);
+    psLogMsg ("psphot.pspsf", PS_LOG_INFO, "psf model %s, ApResid: %f +/- %f (%d x %d model)\n", modelName, psf->ApResid, psf->dApResid, psf->trendNx, psf->trendNy);
 
     psFree (options);

branches/eam_branches/ipp-20140206/psphot/src/psphotCleanup.c

@@ -22 +22 @@
     pmVisualCleanup ();
     psLibFinalize();
+#if (PS_TRACE_ON)
+    // don't display memory leaks unless trace is on. 
     // fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "psphot");
     fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, stdout, false), "psphot");
+#endif
     return;
 }

branches/eam_branches/ipp-20140206/psphot/src/psphotReadout.c

@@ -340 +340 @@
     // calculate source magnitudes
     if (!psphotMagnitudes(config, view, filerule)) { // pass 1 (detections->allSources)
-      psErrorStackPrint(stderr, "Unable to do magnitudes.");
+        psErrorStackPrint(stderr, "Unable to do magnitudes.");
         psErrorClear();
     }
+
+    // calculate lensing parameters
+    if (!psphotLensing(config, view, filerule)) {
+        psErrorStackPrint(stderr, "Unable to do lensing parameters.");
+        psErrorClear();
+    }
+
     if (!psphotEfficiency(config, view, filerule)) { // pass 1
         psErrorStackPrint(stderr, "Unable to determine detection efficiencies from fake sources");
@@ -353 +360 @@
     // replace background in residual image
     if (!psphotSkyReplace (config, view, filerule)) { // pass 1
-      psErrorStackPrint(stderr, "Unable to replace sky");
-      psErrorClear();
+        psErrorStackPrint(stderr, "Unable to replace sky");
+        psErrorClear();
     }
 
     // drop the references to the image pixels held by each source
     if (!psphotSourceFreePixels (config, view, filerule)) { // pass 1
-      psErrorStackPrint(stderr, "Unable to free source pixels");
-      psErrorClear();
+        psErrorStackPrint(stderr, "Unable to free source pixels");
+        psErrorClear();
     }
 

branches/eam_branches/ipp-20140206/psphot/src/psphotStackMatchPSFsNext.c

@@ -105 +105 @@
         psLogMsg ("psphot", PS_LOG_INFO, "target FWHM (%f) is smaller than current FWHM (%f), not smoothing\n", targetFWHM, currentFWHM);
         fwhmValues->data.F32[lastSize + 1] = currentFWHM;
+        pmReadoutMaskInvalid(readout, maskVal, maskSat);
         return false;
     }

branches/eam_branches/ipp-20140206/psphot/src/psphotStackReadout.c

@@ -391 +391 @@
 
                 // re-measure the PSF for the smoothed image (using entries in 'allSources')
-                psphotChoosePSF (config, view, STACK_OUT, false);
+                if (!psphotChoosePSF (config, view, STACK_OUT, false)) {
+                    psLogMsg ("psphot", 3, "failure to construct a psf model in radial aperture loop for entry :%d", entry);
+                    return psphotReadoutCleanup (config, view, STACK_RAW);
+                }
 
                 // this is necessary to update the models based on the new PSF

branches/eam_branches/ipp-20140206/pstamp/scripts/pstamp_insert_request.pl

@@ -45 +45 @@
 my $pstamptool = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $fields = can_run('fields')  or (warn "Can't find fields"  and $missing_tools = 1);
+my $fhead = can_run('fhead')  or (warn "Can't find fhead"  and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -56 +57 @@
     # Note that if it's a pstamp request then REQ_NAME should be defined.
     # if it's a detectability query it will not have a REQ_NAME but will have a QUERY_ID
-    my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
+    # my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
+    my $command = "$fhead -x 0 $tmp_req_file";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-if (0) {
-    # stoopid fields doesn't set exit status to zero when it works
     unless ($success) {
         print STDERR @$stderr_buf;
         exit $error_code >> 8;
     }
-}
     my $output = join "", @$stdout_buf;
-    (undef, $extname, $extver, $req_name) = split " ", $output;
+
+
+    my $makehash = 0;
+    my %hash;
+    foreach my $line (split "\n", $output) {
+        chomp $line;
+        # split lines inte left and right using equals sign. left is the keyword
+        my ($key, $right) = split "=", $line;
+            # skip if there was no '='
+        next if !$right;
+
+        $key =~ s/ //g;
+
+        # separate value from comment
+        my ($value, $comment) = split "/", $right;
+        # remove ' and space characters from key and value
+        $value =~ s/'//g;
+        $value =~ s/ //g;
+
+        #    print "$key $value\n";
+
+        # extract the values that we are looking for
+        $extname  = $value if ($key eq "EXTNAME");
+        $extver   = $value if ($key eq "EXTVER");
+        $req_name = $value if ($key eq "REQ_NAME");
+        $req_name = $value if ($key eq "QUERY_ID");
+
+        # optionally build hash. Duplicate keys get the last value seen
+        if ($makehash) {
+            $hash{$key} = $value;
+        }
+    }
+
+    # (undef, $extname, $extver, $req_name) = split " ", $output;
+
     if (!$extname or ! (($extname eq "PS1_PS_REQUEST") or ($extname eq "MOPS_DETECTABILITY_QUERY"))) {
         print STDERR "invalid request file\n";
+        print "invalid request file\n";
         exit $PS_EXIT_DATA_ERROR;
     }
+
     if (!defined $req_name) {
         print STDERR "invalid request file no REQ_NAME or QUERY_ID\n";
+        print "invalid request file no REQ_NAME or QUERY_ID\n";
+        exit $PS_EXIT_DATA_ERROR;
+    }
+    if (!defined $extver) {
+        print STDERR "invalid request file no EXTVER found\n";
+        print "invalid request file no EXTVER found\n";
         exit $PS_EXIT_DATA_ERROR;
     }
@@ -119 +160 @@
 
 exit 0;
+
 # Ask the database for the next web request number
 sub get_webreq_num

branches/eam_branches/ipp-20140206/pstamp/scripts/pstampparse.pl

@@ -243 +243 @@
 my $watch_for_big_requests = (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/));
 
-my $big_limit = 100;    # XXX: this should be in a configuration file somewhere not hard coded
+# XXX: these should be in a configuration file somewhere not hard coded
+my $big_limit = 100;
+my $job_big_limit = 400;
 
 if ($watch_for_big_requests and $nRows > $big_limit) {
@@ -267 +269 @@
 
     # see whether number of jobs limit for high priority request was exceeded
-    if ($watch_for_big_requests and $num_jobs > $big_limit) {
+    if ($watch_for_big_requests and $num_jobs > $job_big_limit) {
         $label = change_to_lower_priority_label($label);
         $watch_for_big_requests = 0;
@@ -311 +313 @@
     my $rownum   = $row->{ROWNUM};
     if (!validID($rownum)) {
+        $rownum = 'NULL' if !defined $rownum;
         print STDERR "$rownum is not a valid ROWNUM\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -316 +319 @@
     }
     my $job_type = $row->{JOB_TYPE};
-    if (($job_type ne "stamp") and ($job_type ne "get_image")) {
+    if (!defined $job_type || (($job_type ne "stamp") and ($job_type ne "get_image"))) {
+        $job_type = 'NULL' if !defined $job_type;
         print STDERR "$job_type is not a valid JOB_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -325 +329 @@
     if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
         ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
+        $req_type = 'NULL' if !defined $req_type;
         print STDERR "$req_type is not a valid REQ_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);

branches/eam_branches/ipp-20140206/tools/dlapstacks

@@ -14 +14 @@
 
 my $label;
-my $label1  = 'LAP.ThreePi.20120706';
-my $label2  = 'LAP.ThreePi.20110809';
+my $label1  = 'LAP.ThreePi.20130717';
+my $label2  = 'LAP.ThreePi.20120706';
 my $tess_id = 'RINGS.V3';
 

branches/eam_branches/ipp-20140206/tools/fixburntool

@@ -107 +107 @@
 {
     my $command = "$burntool --exp_id $exp_id --class_id $class_id --this_uri $uri --previous_uri $prev_uri";
+    $command .= " --verbose" if $verbose;
     print "$command\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

@@ -48 +48 @@
 releasetool
 fftool
+remotetool
+