Changeset 36928

Timestamp:

Jun 20, 2014, 3:53:40 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20140610

Files:

• . (modified) (1 prop)
• Ohana/src/getstar (modified) (1 prop)
• dbconfig/cam.md (modified) (1 diff)
• dbconfig/changes.txt (modified) (1 diff)
• ippScripts/scripts/queuestaticsky.pl (modified) (1 prop)
• ippScripts/scripts/sc_prepare_stack.pl (modified) (1 diff)
• ippScripts/scripts/sc_transfer_tool.pl (modified) (3 diffs)
• ippTasks/pstamp.pro (modified) (4 diffs)
• ippTools/share/camtool_find_processedexp.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff)
• ippTools/share/remotetool_definebyquery_warp.sql (modified) (1 diff)
• ippTools/src/camtool.c (modified) (2 diffs)
• ippTools/src/camtoolConfig.c (modified) (1 diff)
• ippTools/src/remotetool.c (modified) (5 diffs)
• ippconfig (modified) (1 prop)
• ippconfig/gpc1/psastro.config (modified) (1 diff)
• ippconfig/recipes/ppStats.config (modified) (1 diff)
• ippconfig/recipes/ppStatsFromMetadata.config (modified) (1 diff)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)
• ppStack/src/ppStackReadout.c (modified) (3 diffs)
• ppStats/src/ppStatsFromMetadataStats.c (modified) (3 diffs)
• psModules (modified) (1 prop)
• psastro/src/psastroChipAstrom.c (modified) (1 diff)
• psastro/src/psastroOneChipFit.c (modified) (1 diff)
• psphot/test/tap_psphot_galaxies.pro (modified) (1 diff)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (1 prop)
• pstamp/scripts/pstamp_finish.pl (modified) (1 prop)
• pstamp/scripts/pstampparse.pl (modified) (1 prop)

branches/eam_branches/ipp-20140610/dbconfig/cam.md

@@ -119 +119 @@
     quality        S16      0
     background_model  S16   0
+    astrom_chips   S64      0
 END
 

branches/eam_branches/ipp-20140610/dbconfig/changes.txt

@@ -2521 +2521 @@
 UPDATE dbversion set schema_version = '1.1.79', updated= CURRENT_TIMESTAMP();
 
-
+ALTER TABLE camProcessedExp ADD COLUMN astrom_chips BIGINT;
+UPDATE dbversion set schema_version = '1.1.79', updated= CURRENT_TIMESTAMP();
+
+

branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_prepare_stack.pl

@@ -182 +182 @@
     $ppstack_command    .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK";
     $ppstack_command    .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log ";
-    $ppstack_command    .= " -threads 10 ";
+    $ppstack_command    .= " -threads 4 ";
     $ppstack_command    .= " -dumpconfig ${remote_outroot}.mdc ";
     $ppstack_command    .= " -stack_id $stack_id -skycell_id $skycell_id -tess_id $tess_id ";

branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_transfer_tool.pl

@@ -51 +51 @@
 while(<I>) {
     chomp;
+    $i = int(rand($#filehandles));
+
     my $fline;
     if ($fetch) {
@@ -57 +59 @@
     if (($fetch)&&(!(-e $_))) {  # We are fetching, and do not already have this file.
         print { $filehandles[$i] } $fline;
-        $i++;
+#       $i++;
     }
     elsif (!($fetch)) { # We are pushing
 #       if (-e "${local_tmp}/$_") { 
             print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
-            $i++;
+#           $i++;
 #       }
 #       else { # But somehow couldn't find the file we expected
@@ -69 +71 @@
 #       }
     }
-    if ($i >= $threads) { $i = 0; }
+#    if ($i >= $threads) { $i = 0; }
 }
 close(I);

branches/eam_branches/ipp-20140610/ippTasks/pstamp.pro

@@ -175 +175 @@
 macro pstamp.find.off
     task pstamp.request.find
+        active false
+    end
+end
+macro pstamp.parse.on
+    task pstamp.request.load
+        active true
+    end
+    task pstamp.request.run
+        active true
+    end
+end
+macro pstamp.parse.off
+    task pstamp.request.load
+        active false
+    end
+    task pstamp.request.run
+        active false
+    end
+end
+macro pstamp.job.on
+    task pstamp.job.load
+        active true
+    end
+    task pstamp.job.run
+        active true
+    end
+end
+macro pstamp.job.off
+    task pstamp.job.load
+        active false
+    end
+    task pstamp.job.run
+        active false
+    end
+end
+macro pstamp.finish.on
+    task pstamp.finish.load
+        active true
+    end
+    task pstamp.finish.run
+        active true
+    end
+end
+macro pstamp.finish.off
+    task pstamp.finish.load
+        active false
+    end
+    task pstamp.finish.run
         active false
     end
@@ -469 +517 @@
         add_poll_args run
         add_poll_labels run
+        # limit query for finished requests to small number
+        # because we only run 4 at a time. Using the default queue
+        # depth fouls up the priority orer
+        # XXX: use a varaible
+        $run = $run -limit 8
         command $run
     end
@@ -499 +552 @@
     periods     -timeout 2400
     host        anyhost
+    npending    4
 
     task.exec
@@ -951 +1005 @@
     # overloading nfs
 
-    npending    5
+    npending    2
 
     task.exec

branches/eam_branches/ipp-20140610/ippTools/share/camtool_find_processedexp.sql

@@ -1 +1 @@
 SELECT
     camProcessedExp.*,
+    HEX(IFNULL(camProcessedExp.astrom_chips, 0)) AS good_astrom_chips,
     camRun.workdir,
     camRun.label,

branches/eam_branches/ipp-20140610/ippTools/share/pxadmin_create_tables.sql

@@ -523 +523 @@
     quality SMALLINT NOT NULL DEFAULT 0,
     background_model SMALLINT,
+    astrom_chips BIGINT,
     PRIMARY KEY(cam_id),
     KEY(fault),

branches/eam_branches/ipp-20140610/ippTools/share/remotetool_definebyquery_warp.sql

@@ -1 @@
-SELECT 
-       warp_id AS stage_id
-FROM warpRun 
-       LEFT JOIN remoteComponent ON (remoteComponent.stage_id = warpRun.warp_id) 
-       LEFT JOIN remoteRun ON (
-            remoteComponent.remote_id = remoteRun.remote_id AND 
-            remoteRun.label = warpRun.label
-            )
-WHERE
-warpRun.state = 'new' AND
-(remoteRun.stage = 'warp' OR remoteRun.state IS NULL) AND
-remoteComponent.remote_id IS NULL
+SELECT warp_id AS stage_id FROM (
+SELECT warp_id, count(warpSkyCellMap.skycell_id) AS N
+    FROM warpRun
+    LEFT JOIN warpSkyCellMap USING(warp_id)
+    LEFT JOIN remoteComponent ON (remoteComponent.stage_id = warpRun.warp_id)
+    LEFT JOIN remoteRun ON (remoteComponent.remote_id = remoteRun.remote_id
+                        AND remoteRun.label = warpRun.label             )
+WHERE warpRun.state = 'new'
+AND (remoteRun.stage = 'warp' OR remoteRun.state IS NULL)
+AND remoteComponent.remote_id IS NULL
+-- where hook %s
+GROUP BY warp_id
+) AS warpTry
+WHERE N != 0
+

branches/eam_branches/ipp-20140610/ippTools/src/camtool.c

@@ -501 +501 @@
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
     PXOPT_LOOKUP_S16(background_model, config->args, "-background_model", false, false);
+    PXOPT_LOOKUP_S64(astrom_chips, config->args, "-astrom_chips", false, false);
+
     
     PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
@@ -692 +694 @@
         deteff_uq,
         quality,
-        background_model
+        background_model,
+        astrom_chips
         );
 

branches/eam_branches/ipp-20140610/ippTools/src/camtoolConfig.c

@@ -205 +205 @@
     psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_uq", 0, "define deteff_uq", NAN);
     psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-background_model", 0, "set background_model value", 0);
+    psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-astrom_chips", 0, "chips with successful astrom", -1);
 
     // -processedexp

branches/eam_branches/ipp-20140610/ippTools/src/remotetool.c

@@ -95 +95 @@
   // Get a list of the things we can insert into a new component
   psString query = NULL;
+  psString whereOption = psStringCopy("");
   if (!strcmp(stage,"chip")) {
     query = pxDataGet("remotetool_definebyquery_chip.sql");
@@ -125 +126 @@
     
     if (label) {
-      psStringAppend(&query, " AND (warpRun.label = '%s')", label);
+      // Because warp has to check to see if something other than the state is set correctly, I have to use this wonky way to pass the label in.
+      //      psStringAppend(&query, " AND (warpRun.label = '%s')", label);
+      psStringAppend(&whereOption, "\n AND (warpRun.label = '%s')", label);
     }
   }    
@@ -149 +152 @@
     psFree(limitString);
   }
-  if (!p_psDBRunQuery(config->dbh, query)) {
+  if (!p_psDBRunQueryF(config->dbh, query,whereOption)) {
     psError(PS_ERR_UNKNOWN, false, "database error");
     psFree(query);
@@ -313 +316 @@
 {
   PS_ASSERT_PTR_NON_NULL(config, false);
-  // PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
+  PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
+  psAssert(remote_id,"This should have failed before this point.");
   
   psMetadata *where = psMetadataAlloc();
@@ -341 +345 @@
 {
   PS_ASSERT_PTR_NON_NULL(config, false);
-  // PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
-  
+  PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
+  psAssert(remote_id,"This should have failed before this point.");
+
   psMetadata *where = psMetadataAlloc();
   // Wheres

branches/eam_branches/ipp-20140610/ippconfig/gpc1/psastro.config

@@ -493 +493 @@
    ZERO.POINT.USE.MEAN             BOOL  TRUE
    PSASTRO.FIELD.PADDING           F32   0.01   # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels
-   PSASTRO.MAX.NRAW                S32   1000   # include more than MAX.NREF to account for extras like false-positives
-   PSASTRO.MAX.NREF                S32    500   # if edge skycell ~10% then try ~50 to match 
+   PSASTRO.MAX.NRAW                S32   1500   # include more than MAX.NREF to account for extras like false-positives
+   PSASTRO.MAX.NREF                S32   1000   # if edge skycell ~10% then try ~50 to match. increase to help odd bulk offsets
    PSASTRO.MIN.INST.MAG.RAW        F32   -25.0  # 
    PSASTRO.MAX.INST.MAG.RAW        F32   -5.0

branches/eam_branches/ipp-20140610/ippconfig/recipes/ppStats.config

@@ -116 +116 @@
   HEADER        STR     DT_ASTR  # elapsed time in astrometry processing
   HEADER        STR     NASTRO
+  HEADER        STR     ASTROM_CHIPS
 
   HEADER        STR     ZPT_OBS

branches/eam_branches/ipp-20140610/ippconfig/recipes/ppStatsFromMetadata.config

@@ -274 +274 @@
   ENTRY  VAL  DT_ASTR             F32  SUM              -dtime_astrom      
   ENTRY  VAL  NASTRO              S32  SUM              -n_astrom          
+  ENTRY  VAL  ASTROM_CHIPS        U64  BITWISEOR        -astrom_chips
 
   ENTRY  VAL  ZPT_OBS             F32  SAMPLE_MEAN      -zpt_obs

branches/eam_branches/ipp-20140610/ppStack/src/ppStackReadout.c

@@ -188 +188 @@
     psFree(stack);
 
-    psLogMsg("ppStack", PS_LOG_INFO, "initial stack image sectionNum %d", sectionNum);
+    //MEH change to trace
+    //psLogMsg("ppStack", PS_LOG_INFO, "initial stack image sectionNum %d", sectionNum);
 
     sectionNum++;
@@ -275 +276 @@
         //MEH -- apply bscale offset before norm   
         if (bscaleApplyOffset) {
-            psLogMsg("ppStack", PS_LOG_INFO, "bscaleApplyOffset: %d %f", i, bscaleApplyOffset->data.F32[i]);
+            //MEH change to trace
+            //psLogMsg("ppStack", PS_LOG_INFO, "bscaleApplyOffset: %d %f", i, bscaleApplyOffset->data.F32[i]);
             psBinaryOp(ro->image, ro->image, "-", psScalarAlloc(bscaleApplyOffset->data.F32[i], PS_TYPE_F32));
         }
@@ -307 +309 @@
     psFree(stack);
 
-    psLogMsg("ppStack", PS_LOG_INFO, "final stack image sectionNum %d", sectionNum);
+    //MEH change to trace
+    //psLogMsg("ppStack", PS_LOG_INFO, "final stack image sectionNum %d", sectionNum);
 
     sectionNum++;

branches/eam_branches/ipp-20140610/ppStats/src/ppStatsFromMetadataStats.c

@@ -1 +1 @@
 # include "ppStatsInternal.h"
+
+static void computeBitwiseOr(ppStatsEntry *entry);
 
 // calculate the stats for the non-constant entries (already calculated)
@@ -11 +13 @@
         // XXX skip or warn on missing stats?
         if (!entry->vector) continue;
+
+        if (!strcasecmp (entry->statistic, "bitwiseor")) {
+            computeBitwiseOr(entry);
+            continue;
+        }
 
         psStatsOptions option;
@@ -68 +75 @@
     return true;
 }
+
+static void computeBitwiseOr(ppStatsEntry *entry)
+{
+    psU64 result = 0;
+    psVector *vector = entry->vector;
+    for (int j = 0; j < vector->n; j++) {
+        // XXX: should we handle other types
+        if (entry->type == PS_DATA_U64) {
+            result |= vector->data.U64[j];
+        } else if (entry->type == PS_DATA_U32) {
+            result |= vector->data.U32[j];
+        } else {
+            return;
+        } 
+    }
+
+    if (entry->type == PS_DATA_U64) {
+        entry->value = psMetadataItemAllocU64(entry->keyword, entry->statistic, result);
+    } else {
+        entry->value = psMetadataItemAllocU32(entry->keyword, entry->statistic, result);
+    }
+
+    return;
+}
+

branches/eam_branches/ipp-20140610/psastro/src/psastroChipAstrom.c

@@ -143 +143 @@
                 numGoodRO++;
 
+                psU64 astrom_chip_val = 1;
+                astrom_chip_val <<= view->chip;
+                psMetadataAddU64 (updates, PS_LIST_TAIL, "ASTROM_CHIPS", PS_META_REPLACE, 
+                    "chips that passed astrometry", astrom_chip_val);
+
                 // write the elapsed time here; this will be updated in psastroMosaicAstrometry, if called
                 psMetadataAddF32 (updates, PS_LIST_TAIL, "DT_ASTR", PS_META_REPLACE, "elapsed psastro time", psTimerMark ("psastroAnalysis"));

branches/eam_branches/ipp-20140610/psastro/src/psastroOneChipFit.c

@@ -228 +228 @@
     psMetadataAddF32 (updates, PS_LIST_TAIL, "CPRECISE", PS_META_REPLACE, "astrometry precision (arcsec)", 0.0);
     psMetadataAddS32 (updates, PS_LIST_TAIL, "NASTRO",   PS_META_REPLACE, "number of astrometry stars", 0);
+    psMetadataAddU64 (updates, PS_LIST_TAIL, "ASTROM_CHIPS", PS_META_REPLACE, "chips that passed astrometry", 0);
     return true;
 }

branches/eam_branches/ipp-20140610/psphot/test/tap_psphot_galaxies.pro

@@ -194 +194 @@
   $psphotConfig = $psphotConfig -Di PSPHOT:LMM_FIT_GAIN_FACTOR_MODE 2
   $psphotConfig = $psphotConfig -Db PSPHOT:SAVE.RESID T
-  # $psphotConfig = $psphotConfig -D  PSPHOT:PSF_MODEL PS_MODEL_PS1_V1
-  $psphotConfig = $psphotConfig -D  PSPHOT:PSF_MODEL PS_MODEL_GAUSS
+  $psphotConfig = $psphotConfig -D  PSPHOT:PSF_MODEL PS_MODEL_PS1_V1
+  # $psphotConfig = $psphotConfig -D  PSPHOT:PSF_MODEL PS_MODEL_GAUSS
   $psphotConfig = $psphotConfig -D  PSPHOT:EXTENDED_SOURCE_MODELS_SELECTION $fitModel
-  $psphotConfig = $psphotConfig -D  PSPHOT:OUTPUT.FORMAT PS1_V5
+  # $psphotConfig = $psphotConfig -D  PSPHOT:OUTPUT.FORMAT PS1_V5
+  $psphotConfig = $psphotConfig -D  PSPHOT:OUTPUT.FORMAT PS1_SV3
+  $psphotConfig = $psphotConfig -Db PSPHOT:LENSING_PARAMETERS T
 
   # ppImage / psphot on the output