Index: /branches/eam_branches/ipp-20140610/dbconfig/cam.md =================================================================== --- /branches/eam_branches/ipp-20140610/dbconfig/cam.md (revision 36927) +++ /branches/eam_branches/ipp-20140610/dbconfig/cam.md (revision 36928) @@ -119,4 +119,5 @@ quality S16 0 background_model S16 0 + astrom_chips S64 0 END Index: /branches/eam_branches/ipp-20140610/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/ipp-20140610/dbconfig/changes.txt (revision 36927) +++ /branches/eam_branches/ipp-20140610/dbconfig/changes.txt (revision 36928) @@ -2521,3 +2521,6 @@ UPDATE dbversion set schema_version = '1.1.79', updated= CURRENT_TIMESTAMP(); - +ALTER TABLE camProcessedExp ADD COLUMN astrom_chips BIGINT; +UPDATE dbversion set schema_version = '1.1.79', updated= CURRENT_TIMESTAMP(); + + Index: /branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_prepare_stack.pl =================================================================== --- /branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_prepare_stack.pl (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_prepare_stack.pl (revision 36928) @@ -182,5 +182,5 @@ $ppstack_command .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK"; $ppstack_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; - $ppstack_command .= " -threads 10 "; + $ppstack_command .= " -threads 4 "; $ppstack_command .= " -dumpconfig ${remote_outroot}.mdc "; $ppstack_command .= " -stack_id $stack_id -skycell_id $skycell_id -tess_id $tess_id "; Index: /branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_transfer_tool.pl =================================================================== --- /branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_transfer_tool.pl (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippScripts/scripts/sc_transfer_tool.pl (revision 36928) @@ -51,4 +51,6 @@ while() { chomp; + $i = int(rand($#filehandles)); + my $fline; if ($fetch) { @@ -57,10 +59,10 @@ if (($fetch)&&(!(-e $_))) { # We are fetching, and do not already have this file. print { $filehandles[$i] } $fline; - $i++; +# $i++; } elsif (!($fetch)) { # We are pushing # if (-e "${local_tmp}/$_") { print { $filehandles[$i] } "${local_tmp}/$_" . "\n"; - $i++; +# $i++; # } # else { # But somehow couldn't find the file we expected @@ -69,5 +71,5 @@ # } } - if ($i >= $threads) { $i = 0; } +# if ($i >= $threads) { $i = 0; } } close(I); Index: /branches/eam_branches/ipp-20140610/ippTasks/pstamp.pro =================================================================== --- /branches/eam_branches/ipp-20140610/ippTasks/pstamp.pro (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippTasks/pstamp.pro (revision 36928) @@ -175,4 +175,52 @@ macro pstamp.find.off task pstamp.request.find + active false + end +end +macro pstamp.parse.on + task pstamp.request.load + active true + end + task pstamp.request.run + active true + end +end +macro pstamp.parse.off + task pstamp.request.load + active false + end + task pstamp.request.run + active false + end +end +macro pstamp.job.on + task pstamp.job.load + active true + end + task pstamp.job.run + active true + end +end +macro pstamp.job.off + task pstamp.job.load + active false + end + task pstamp.job.run + active false + end +end +macro pstamp.finish.on + task pstamp.finish.load + active true + end + task pstamp.finish.run + active true + end +end +macro pstamp.finish.off + task pstamp.finish.load + active false + end + task pstamp.finish.run active false end @@ -469,4 +517,9 @@ add_poll_args run add_poll_labels run + # limit query for finished requests to small number + # because we only run 4 at a time. Using the default queue + # depth fouls up the priority orer + # XXX: use a varaible + $run = $run -limit 8 command $run end @@ -499,4 +552,5 @@ periods -timeout 2400 host anyhost + npending 4 task.exec @@ -951,5 +1005,5 @@ # overloading nfs - npending 5 + npending 2 task.exec Index: /branches/eam_branches/ipp-20140610/ippTools/share/camtool_find_processedexp.sql =================================================================== --- /branches/eam_branches/ipp-20140610/ippTools/share/camtool_find_processedexp.sql (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippTools/share/camtool_find_processedexp.sql (revision 36928) @@ -1,4 +1,5 @@ SELECT camProcessedExp.*, + HEX(IFNULL(camProcessedExp.astrom_chips, 0)) AS good_astrom_chips, camRun.workdir, camRun.label, Index: /branches/eam_branches/ipp-20140610/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20140610/ippTools/share/pxadmin_create_tables.sql (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippTools/share/pxadmin_create_tables.sql (revision 36928) @@ -523,4 +523,5 @@ quality SMALLINT NOT NULL DEFAULT 0, background_model SMALLINT, + astrom_chips BIGINT, PRIMARY KEY(cam_id), KEY(fault), Index: /branches/eam_branches/ipp-20140610/ippTools/share/remotetool_definebyquery_warp.sql =================================================================== --- /branches/eam_branches/ipp-20140610/ippTools/share/remotetool_definebyquery_warp.sql (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippTools/share/remotetool_definebyquery_warp.sql (revision 36928) @@ -1,12 +1,15 @@ -SELECT - warp_id AS stage_id -FROM warpRun - LEFT JOIN remoteComponent ON (remoteComponent.stage_id = warpRun.warp_id) - LEFT JOIN remoteRun ON ( - remoteComponent.remote_id = remoteRun.remote_id AND - remoteRun.label = warpRun.label - ) -WHERE -warpRun.state = 'new' AND -(remoteRun.stage = 'warp' OR remoteRun.state IS NULL) AND -remoteComponent.remote_id IS NULL +SELECT warp_id AS stage_id FROM ( +SELECT warp_id, count(warpSkyCellMap.skycell_id) AS N + FROM warpRun + LEFT JOIN warpSkyCellMap USING(warp_id) + LEFT JOIN remoteComponent ON (remoteComponent.stage_id = warpRun.warp_id) + LEFT JOIN remoteRun ON (remoteComponent.remote_id = remoteRun.remote_id + AND remoteRun.label = warpRun.label ) +WHERE warpRun.state = 'new' +AND (remoteRun.stage = 'warp' OR remoteRun.state IS NULL) +AND remoteComponent.remote_id IS NULL +-- where hook %s +GROUP BY warp_id +) AS warpTry +WHERE N != 0 + Index: /branches/eam_branches/ipp-20140610/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/ipp-20140610/ippTools/src/camtool.c (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippTools/src/camtool.c (revision 36928) @@ -501,4 +501,6 @@ PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); PXOPT_LOOKUP_S16(background_model, config->args, "-background_model", false, false); + PXOPT_LOOKUP_S64(astrom_chips, config->args, "-astrom_chips", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); @@ -692,5 +694,6 @@ deteff_uq, quality, - background_model + background_model, + astrom_chips ); Index: /branches/eam_branches/ipp-20140610/ippTools/src/camtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20140610/ippTools/src/camtoolConfig.c (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippTools/src/camtoolConfig.c (revision 36928) @@ -205,4 +205,5 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_uq", 0, "define deteff_uq", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-background_model", 0, "set background_model value", 0); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-astrom_chips", 0, "chips with successful astrom", -1); // -processedexp Index: /branches/eam_branches/ipp-20140610/ippTools/src/remotetool.c =================================================================== --- /branches/eam_branches/ipp-20140610/ippTools/src/remotetool.c (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippTools/src/remotetool.c (revision 36928) @@ -95,4 +95,5 @@ // Get a list of the things we can insert into a new component psString query = NULL; + psString whereOption = psStringCopy(""); if (!strcmp(stage,"chip")) { query = pxDataGet("remotetool_definebyquery_chip.sql"); @@ -125,5 +126,7 @@ if (label) { - psStringAppend(&query, " AND (warpRun.label = '%s')", label); + // Because warp has to check to see if something other than the state is set correctly, I have to use this wonky way to pass the label in. + // psStringAppend(&query, " AND (warpRun.label = '%s')", label); + psStringAppend(&whereOption, "\n AND (warpRun.label = '%s')", label); } } @@ -149,5 +152,5 @@ psFree(limitString); } - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query,whereOption)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -313,5 +316,6 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - // PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + psAssert(remote_id,"This should have failed before this point."); psMetadata *where = psMetadataAlloc(); @@ -341,6 +345,7 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - // PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); - + PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + psAssert(remote_id,"This should have failed before this point."); + psMetadata *where = psMetadataAlloc(); // Wheres Index: /branches/eam_branches/ipp-20140610/ippconfig/gpc1/psastro.config =================================================================== --- /branches/eam_branches/ipp-20140610/ippconfig/gpc1/psastro.config (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippconfig/gpc1/psastro.config (revision 36928) @@ -493,6 +493,6 @@ ZERO.POINT.USE.MEAN BOOL TRUE PSASTRO.FIELD.PADDING F32 0.01 # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels - PSASTRO.MAX.NRAW S32 1000 # include more than MAX.NREF to account for extras like false-positives - PSASTRO.MAX.NREF S32 500 # if edge skycell ~10% then try ~50 to match + PSASTRO.MAX.NRAW S32 1500 # include more than MAX.NREF to account for extras like false-positives + PSASTRO.MAX.NREF S32 1000 # if edge skycell ~10% then try ~50 to match. increase to help odd bulk offsets PSASTRO.MIN.INST.MAG.RAW F32 -25.0 # PSASTRO.MAX.INST.MAG.RAW F32 -5.0 Index: /branches/eam_branches/ipp-20140610/ippconfig/recipes/ppStats.config =================================================================== --- /branches/eam_branches/ipp-20140610/ippconfig/recipes/ppStats.config (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippconfig/recipes/ppStats.config (revision 36928) @@ -116,4 +116,5 @@ HEADER STR DT_ASTR # elapsed time in astrometry processing HEADER STR NASTRO + HEADER STR ASTROM_CHIPS HEADER STR ZPT_OBS Index: /branches/eam_branches/ipp-20140610/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- /branches/eam_branches/ipp-20140610/ippconfig/recipes/ppStatsFromMetadata.config (revision 36927) +++ /branches/eam_branches/ipp-20140610/ippconfig/recipes/ppStatsFromMetadata.config (revision 36928) @@ -274,4 +274,5 @@ ENTRY VAL DT_ASTR F32 SUM -dtime_astrom ENTRY VAL NASTRO S32 SUM -n_astrom + ENTRY VAL ASTROM_CHIPS U64 BITWISEOR -astrom_chips ENTRY VAL ZPT_OBS F32 SAMPLE_MEAN -zpt_obs Index: /branches/eam_branches/ipp-20140610/ppStack/src/ppStackReadout.c =================================================================== --- /branches/eam_branches/ipp-20140610/ppStack/src/ppStackReadout.c (revision 36927) +++ /branches/eam_branches/ipp-20140610/ppStack/src/ppStackReadout.c (revision 36928) @@ -188,5 +188,6 @@ psFree(stack); - psLogMsg("ppStack", PS_LOG_INFO, "initial stack image sectionNum %d", sectionNum); + //MEH change to trace + //psLogMsg("ppStack", PS_LOG_INFO, "initial stack image sectionNum %d", sectionNum); sectionNum++; @@ -275,5 +276,6 @@ //MEH -- apply bscale offset before norm if (bscaleApplyOffset) { - psLogMsg("ppStack", PS_LOG_INFO, "bscaleApplyOffset: %d %f", i, bscaleApplyOffset->data.F32[i]); + //MEH change to trace + //psLogMsg("ppStack", PS_LOG_INFO, "bscaleApplyOffset: %d %f", i, bscaleApplyOffset->data.F32[i]); psBinaryOp(ro->image, ro->image, "-", psScalarAlloc(bscaleApplyOffset->data.F32[i], PS_TYPE_F32)); } @@ -307,5 +309,6 @@ psFree(stack); - psLogMsg("ppStack", PS_LOG_INFO, "final stack image sectionNum %d", sectionNum); + //MEH change to trace + //psLogMsg("ppStack", PS_LOG_INFO, "final stack image sectionNum %d", sectionNum); sectionNum++; Index: /branches/eam_branches/ipp-20140610/ppStats/src/ppStatsFromMetadataStats.c =================================================================== --- /branches/eam_branches/ipp-20140610/ppStats/src/ppStatsFromMetadataStats.c (revision 36927) +++ /branches/eam_branches/ipp-20140610/ppStats/src/ppStatsFromMetadataStats.c (revision 36928) @@ -1,3 +1,5 @@ # include "ppStatsInternal.h" + +static void computeBitwiseOr(ppStatsEntry *entry); // calculate the stats for the non-constant entries (already calculated) @@ -11,4 +13,9 @@ // XXX skip or warn on missing stats? if (!entry->vector) continue; + + if (!strcasecmp (entry->statistic, "bitwiseor")) { + computeBitwiseOr(entry); + continue; + } psStatsOptions option; @@ -68,2 +75,27 @@ return true; } + +static void computeBitwiseOr(ppStatsEntry *entry) +{ + psU64 result = 0; + psVector *vector = entry->vector; + for (int j = 0; j < vector->n; j++) { + // XXX: should we handle other types + if (entry->type == PS_DATA_U64) { + result |= vector->data.U64[j]; + } else if (entry->type == PS_DATA_U32) { + result |= vector->data.U32[j]; + } else { + return; + } + } + + if (entry->type == PS_DATA_U64) { + entry->value = psMetadataItemAllocU64(entry->keyword, entry->statistic, result); + } else { + entry->value = psMetadataItemAllocU32(entry->keyword, entry->statistic, result); + } + + return; +} + Index: /branches/eam_branches/ipp-20140610/psastro/src/psastroChipAstrom.c =================================================================== --- /branches/eam_branches/ipp-20140610/psastro/src/psastroChipAstrom.c (revision 36927) +++ /branches/eam_branches/ipp-20140610/psastro/src/psastroChipAstrom.c (revision 36928) @@ -143,4 +143,9 @@ numGoodRO++; + psU64 astrom_chip_val = 1; + astrom_chip_val <<= view->chip; + psMetadataAddU64 (updates, PS_LIST_TAIL, "ASTROM_CHIPS", PS_META_REPLACE, + "chips that passed astrometry", astrom_chip_val); + // write the elapsed time here; this will be updated in psastroMosaicAstrometry, if called psMetadataAddF32 (updates, PS_LIST_TAIL, "DT_ASTR", PS_META_REPLACE, "elapsed psastro time", psTimerMark ("psastroAnalysis")); Index: /branches/eam_branches/ipp-20140610/psastro/src/psastroOneChipFit.c =================================================================== --- /branches/eam_branches/ipp-20140610/psastro/src/psastroOneChipFit.c (revision 36927) +++ /branches/eam_branches/ipp-20140610/psastro/src/psastroOneChipFit.c (revision 36928) @@ -228,4 +228,5 @@ psMetadataAddF32 (updates, PS_LIST_TAIL, "CPRECISE", PS_META_REPLACE, "astrometry precision (arcsec)", 0.0); psMetadataAddS32 (updates, PS_LIST_TAIL, "NASTRO", PS_META_REPLACE, "number of astrometry stars", 0); + psMetadataAddU64 (updates, PS_LIST_TAIL, "ASTROM_CHIPS", PS_META_REPLACE, "chips that passed astrometry", 0); return true; } Index: /branches/eam_branches/ipp-20140610/psphot/test/tap_psphot_galaxies.pro =================================================================== --- /branches/eam_branches/ipp-20140610/psphot/test/tap_psphot_galaxies.pro (revision 36927) +++ /branches/eam_branches/ipp-20140610/psphot/test/tap_psphot_galaxies.pro (revision 36928) @@ -194,8 +194,10 @@ $psphotConfig = $psphotConfig -Di PSPHOT:LMM_FIT_GAIN_FACTOR_MODE 2 $psphotConfig = $psphotConfig -Db PSPHOT:SAVE.RESID T - # $psphotConfig = $psphotConfig -D PSPHOT:PSF_MODEL PS_MODEL_PS1_V1 - $psphotConfig = $psphotConfig -D PSPHOT:PSF_MODEL PS_MODEL_GAUSS + $psphotConfig = $psphotConfig -D PSPHOT:PSF_MODEL PS_MODEL_PS1_V1 + # $psphotConfig = $psphotConfig -D PSPHOT:PSF_MODEL PS_MODEL_GAUSS $psphotConfig = $psphotConfig -D PSPHOT:EXTENDED_SOURCE_MODELS_SELECTION $fitModel - $psphotConfig = $psphotConfig -D PSPHOT:OUTPUT.FORMAT PS1_V5 + # $psphotConfig = $psphotConfig -D PSPHOT:OUTPUT.FORMAT PS1_V5 + $psphotConfig = $psphotConfig -D PSPHOT:OUTPUT.FORMAT PS1_SV3 + $psphotConfig = $psphotConfig -Db PSPHOT:LENSING_PARAMETERS T # ppImage / psphot on the output