Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/addstar.pro =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/addstar.pro (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/addstar.pro (revision 37823) @@ -52,4 +52,10 @@ active false end + task addstar.revert.diff + active false + end + task addstar.revert.fullforce + active false + end end @@ -65,4 +71,10 @@ end task addstar.revert.skycal + active true + end + task addstar.revert.diff + active true + end + task addstar.revert.fullforce active true end @@ -141,4 +153,6 @@ $addstar_revert_DB_SSM = 0 $addstar_revert_DB_SC = 0 +$addstar_revert_DB_FF = 0 +$addstar_revert_DB_DF = 0 #$LOADEXEC_ADD = 300 @@ -275,4 +289,10 @@ sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID end + if ("$STAGE" == "diff") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "fullforce") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end @@ -293,4 +313,10 @@ if ("$STAGE" == "skycal") $run = $run --stage_id $STAGE_ID + end + if ("$STAGE" == "diff") + $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 + end + if ("$STAGE" == "fullforce") + $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 end @@ -526,2 +552,98 @@ end end + + +task addstar.revert.diff + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage diff + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_DF + $run = $run -dbname $DB:$addstar_revert_DB_DF + $addstar_revert_DB_DF ++ + if ($addstar_revert_DB_DF >= $DB:n) set addstar_revert_DB_DF = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +task addstar.revert.fullforce + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage fullforce + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_FF + $run = $run -dbname $DB:$addstar_revert_DB_FF + $addstar_revert_DB_FF ++ + if ($addstar_revert_DB_FF >= $DB:n) set addstar_revert_DB_FF = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/register.pro =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/register.pro (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/register.pro (revision 37823) @@ -36,4 +36,7 @@ active false end + task register.exp.revert + active false + end task register.exp.load active true @@ -60,4 +63,7 @@ active false end + task register.exp.revert + active false + end task register.exp.load active false @@ -78,8 +84,14 @@ active true end + task register.exp.revert + active true + end end macro register.revert.off task register.imfile.revert + active false + end + task register.exp.revert active false end @@ -202,4 +214,5 @@ $regPendingBurntoolImfile_DB = 0 $regRevertImfile_DB = 0 +$regRevertExp_DB = 0 $regPendingExp_DB = 0 $reg_datePAGE = 0 @@ -371,4 +384,50 @@ $regRevertImfile_DB ++ if ($regRevertImfile_DB >= $DB:n) set regRevertImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out + task.exit timeout + showcommand timeout + end +end + + + +task register.exp.revert + host local + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/register.exp.revert.log + + task.exec +# Only revert fault 2, because other faults likely mean something horrible is happening. + $run = regtool -revertprocessedexp -fault 2 + if ($DB:n == 0) + option DEFAULT + else + option $DB:$regRevertExp_DB + $run = $run -dbname $DB:$regRevertExp_DB + $regRevertExp_DB ++ + if ($regRevertExp_DB >= $DB:n) set regRevertExp_DB = 0 end add_poll_args run Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/survey.pro =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/survey.pro (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTasks/survey.pro (revision 37823) @@ -178,13 +178,14 @@ # user functions to manipulate warp/stack diff labels macro survey.add.WSdiff - if ($0 != 6) - echo "USAGE: survey.add.WSdiff (warp label) (stack label) (dist_group) (workdir base) (target label)" + if ($0 != 7) + echo "USAGE: survey.add.WSdiff (key) (warp label) (stack label) (dist_group) (workdir base) (target label)" break end book newpage SURVEY_DIFF_WARPSTACK $1 - book setword SURVEY_DIFF_WARPSTACK $1 STACK_LABEL $2 - book setword SURVEY_DIFF_WARPSTACK $1 DIST_GROUP $3 - book setword SURVEY_DIFF_WARPSTACK $1 WORKDIR $4 - book setword SURVEY_DIFF_WARPSTACK $1 TARGET_LABEL $5 + book setword SURVEY_DIFF_WARPSTACK $1 WARP_LABEL $2 + book setword SURVEY_DIFF_WARPSTACK $1 STACK_LABEL $3 + book setword SURVEY_DIFF_WARPSTACK $1 DIST_GROUP $4 + book setword SURVEY_DIFF_WARPSTACK $1 WORKDIR $5 + book setword SURVEY_DIFF_WARPSTACK $1 TARGET_LABEL $6 book setword SURVEY_DIFF_WARPSTACK $1 STATE PENDING end @@ -697,4 +698,7 @@ periods -exec $SURVEY_EXEC periods -timeout $SURVEY_TIMEOUT + ## time in UT -- want to start ~0900 HST and run until next night starts. 24h boundary so two trange needed? + trange 19:00:00 23:59:00 + trange 00:00:00 05:00:00 npending 1 @@ -710,13 +714,13 @@ endif - book getpage SURVEY_DIFF_WARPSTACK 0 -var label -key STATE NEW - if ("$label" == "NULL") + book getpage SURVEY_DIFF_WARPSTACK 0 -var wsKey -key STATE NEW + if ("$wsKey" == "NULL") # All labels have been done --- reset # echo "Resetting labels" for i 0 $N - book getpage SURVEY_DIFF_WARPSTACK $i -var label - book setword SURVEY_DIFF_WARPSTACK $label STATE NEW + book getpage SURVEY_DIFF_WARPSTACK $i -var wsKey + book setword SURVEY_DIFF_WARPSTACK $wsKey STATE NEW end - book getpage SURVEY_DIFF_WARPSTACK 0 -var label -key STATE NEW + book getpage SURVEY_DIFF_WARPSTACK 0 -var wsKey -key STATE NEW # Select different database @@ -725,9 +729,10 @@ end - book setword SURVEY_DIFF_WARPSTACK $label STATE DONE - book getword SURVEY_DIFF_WARPSTACK $label STACK_LABEL -var stack_label - book getword SURVEY_DIFF_WARPSTACK $label WORKDIR -var workdir - book getword SURVEY_DIFF_WARPSTACK $label DIST_GROUP -var dist_group - book getword SURVEY_DIFF_WARPSTACK $label TARGET_LABEL -var target_label + book setword SURVEY_DIFF_WARPSTACK $wsKey STATE DONE + book getword SURVEY_DIFF_WARPSTACK $wsKey WARP_LABEL -var warp_label + book getword SURVEY_DIFF_WARPSTACK $wsKey STACK_LABEL -var stack_label + book getword SURVEY_DIFF_WARPSTACK $wsKey WORKDIR -var workdir + book getword SURVEY_DIFF_WARPSTACK $wsKey DIST_GROUP -var dist_group + book getword SURVEY_DIFF_WARPSTACK $wsKey TARGET_LABEL -var target_label $year = `date +%Y` @@ -737,5 +742,5 @@ # Do we need anything else here? $run = difftool -definewarpstack -good_frac 0.2 - $run = $run -warp_label $label + $run = $run -warp_label $warp_label $run = $run -stack_label $stack_label $run = $run -set_dist_group $dist_group @@ -1012,5 +1017,10 @@ $run = $run -uncensored end - + if ("$stage" == "fullforce") + $run = $run -uncensored + end + if ("$stage" == "diff") + $run = $run -uncensored) + end # we need to handle isp/gpc1: gpc1 wants destreaked, isp wants uncensored. Perhaps the best way is by default do destreaked (the most paranoid), and if db = isp (is this the best way?) then do uncensored Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/Makefile.am (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/Makefile.am (revision 37823) @@ -8,4 +8,8 @@ addtool_find_cam_id_dvo.sql \ addtool_find_cam_id.sql \ + addtool_find_diff_id.sql \ + addtool_find_diff_id_dvo.sql \ + addtool_find_ff_id.sql \ + addtool_find_ff_id_dvo.sql \ addtool_find_skycal_id_dvo.sql \ addtool_find_skycal_id.sql \ @@ -16,4 +20,6 @@ addtool_find_pendingexp.sql \ addtool_find_pendingexp_cam.sql \ + addtool_find_pendingexp_diff.sql \ + addtool_find_pendingexp_ff.sql \ addtool_find_pendingexp_skycal.sql \ addtool_find_pendingexp_stack.sql \ @@ -21,4 +27,6 @@ addtool_find_pendingmergeprocess.sql \ addtool_find_processedexp_cam.sql \ + addtool_find_processedexp_diff.sql \ + addtool_find_processedexp_ff.sql \ addtool_find_processedexp_skycal.sql \ addtool_find_processedexp_stack.sql \ @@ -29,4 +37,6 @@ addtool_pendingcleanuprun.sql \ addtool_queue_cam_id.sql \ + addtool_queue_diff_id.sql \ + addtool_queue_ff_id.sql \ addtool_queue_stack_id.sql \ addtool_queue_sky_id_multi.sql \ @@ -35,4 +45,6 @@ addtool_revertminidvodbprocessed.sql \ addtool_revertprocessedexp_cam.sql \ + addtool_revertprocessedexp_diff.sql \ + addtool_revertprocessedexp_ff.sql \ addtool_revertprocessedexp_skycal.sql \ addtool_revertprocessedexp_stack.sql \ @@ -471,4 +483,5 @@ laptool_revertgroup.sql \ laptool_filtersforgroup.sql \ + laptool_revertsass.sql \ vptool_find_rawexp.sql \ vptool_pendingimfile.sql \ @@ -484,5 +497,8 @@ releasetool_definerelstack.sql \ releasetool_definerelstack_with_skycal.sql \ + releasetool_deleterelexp.sql \ + releasetool_deleterelstack.sql \ releasetool_listrelstack.sql \ + releasetool_setrelstackcalibratedfromskycal.sql \ releasetool_summary.sql \ releasetool_definerelgroup_select_lap.sql \ Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_diff_id.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_diff_id.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_diff_id.sql (revision 37823) @@ -0,0 +1,22 @@ +SELECT + diffRun.*, diffInputSkyfile.diff_skyfile_id +FROM + diffInputSkyfile +JOIN diffSkyfile using (diff_id, skycell_id) +JOIN diffRun using (diff_id) + +LEFT JOIN (SELECT diff_id AS added_diff_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN diffRun on (diff_id = stage_id and addRun.stage = 'diff') + ) as foo + ON diff_id = added_diff_id + -- AND stage = 'diff' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + diffRun.state = 'full' + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND added_diff_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_diff_id_dvo.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_diff_id_dvo.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_diff_id_dvo.sql (revision 37823) @@ -0,0 +1,12 @@ +SELECT diffRun.*, diffInputSkyfile.diff_skyfile_id +FROM + diffInputSkyfile +JOIN diffSkyfile using (diff_id, skycell_id) +JOIN diffRun using (diff_id) + +WHERE diffRun.state = 'full' and diffSkyfile.quality = 0 + AND diffRun.diff_id NOT IN (SELECT diffRun.diff_id + FROM addRun + JOIN diffRun on (diffRun.diff_id = addRun.stage_id and addRun.stage = 'diff') JOIN diffSkyfile on (diffRun.diff_id = diffSkyfile.diff_id) + WHERE addRun.stage = 'diff' AND %s + ) Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_ff_id.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_ff_id.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_ff_id.sql (revision 37823) @@ -0,0 +1,21 @@ +SELECT + fullForceRun.*, fullForceResult.warp_id +FROM + fullForceRun +JOIN fullForceResult using (ff_id) + +LEFT JOIN (SELECT ff_id AS added_ff_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN fullForceRun on (ff_id = stage_id and addRun.stage = 'fullforce') + ) as foo + ON ff_id = added_ff_id + --AND stage = 'skycal' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + fullForceRun.state = 'full' + AND fullForceResult.quality = 0 + AND fullForceResult.fault = 0 + AND added_ff_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_ff_id_dvo.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_ff_id_dvo.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_ff_id_dvo.sql (revision 37823) @@ -0,0 +1,8 @@ +SELECT fullForceRun.*, fullForceResult.warp_id FROM fullForceRun JOIN fullForceResult using (ff_id) + +WHERE fullForceRun.state = 'full' and fullForceResult.quality = 0 + AND fullForceRun.ff_id NOT IN (SELECT fullForceRun.ff_id + FROM addRun + JOIN fullForceRun on (fullForceRun.ff_id = addRun.stage_id and addRun.stage = 'fullforce') JOIN fullForceResult on (fullForceRun.ff_id = fullForceResult.ff_id) + WHERE addRun.stage = 'fullforce' AND %s + ) Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_pendingexp_diff.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_pendingexp_diff.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_pendingexp_diff.sql (revision 37823) @@ -0,0 +1,21 @@ + SELECT + addRun.*, + diffSkyfile.path_base as stageroot, + 'GPC1' as camera, + 'PS1' as telescope +FROM addRun +JOIN diffInputSkyfile + ON (diff_id = stage_id and diff_skyfile_id = stage_extra1 and stage = "diff") +JOIN diffSkyfile + USING (diff_id, skycell_id) +JOIN diffRun + using(diff_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + diffRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_pendingexp_ff.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_pendingexp_ff.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_pendingexp_ff.sql (revision 37823) @@ -0,0 +1,19 @@ + SELECT + addRun.*, + fullForceResult.path_base as stageroot, + 'GPC1' as camera, + 'PS1' as telescope +FROM addRun +JOIN fullForceResult + ON (ff_id = stage_id and warp_id = stage_extra1 and stage = "fullforce") +JOIN fullForceRun + using(ff_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + fullForceRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_processedexp_diff.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_processedexp_diff.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_processedexp_diff.sql (revision 37823) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_processedexp_ff.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_processedexp_ff.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_find_processedexp_ff.sql (revision 37823) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_queue_diff_id.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_queue_diff_id.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_queue_diff_id.sql (revision 37823) @@ -0,0 +1,23 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'diff', -- stage + diff_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM diffRun + WHERE + diffRun.state = 'full' + AND diffRun.diff_id = %lld + Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_queue_ff_id.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_queue_ff_id.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_queue_ff_id.sql (revision 37823) @@ -0,0 +1,22 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'fullforce', -- stage + ff_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM fullForceRun + WHERE + fullForceRun.state = 'full' + AND fullForceRun.ff_id = %lld Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_revertprocessedexp_diff.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_revertprocessedexp_diff.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_revertprocessedexp_diff.sql (revision 37823) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'diff' Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_revertprocessedexp_ff.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_revertprocessedexp_ff.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/addtool_revertprocessedexp_ff.sql (revision 37823) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'fullforce' Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/fftool_result.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/fftool_result.sql (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/fftool_result.sql (revision 37823) @@ -11,4 +11,6 @@ stackRun.skycell_id, stackRun.filter, + rawExp.exp_id, + rawExp.exp_name, rawExp.camera FROM fullForceRun Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/laptool_revertsass.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/laptool_revertsass.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/laptool_revertsass.sql (revision 37823) @@ -0,0 +1,1 @@ +UPDATE lapRun SET final_sass_id = NULL,quick_sass_id = NULL Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/pstamptool_pendingdependent.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/pstamptool_pendingdependent.sql (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/pstamptool_pendingdependent.sql (revision 37823) @@ -2,4 +2,5 @@ pstampDependent.*, pstampRequest.label, + MIN(req_id) as first_req_id, concat( pstampDependent.outdir, '/checkdep.', dep_id, '.log') as logfile, IFNULL(Label.priority, 10000) AS priority Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_deleterelexp.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_deleterelexp.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_deleterelexp.sql (revision 37823) @@ -0,0 +1,4 @@ +DELETE FROM relExp USING relExp, ippRelease, rawExp, camRun +WHERE relExp.rel_id = ippRelease.rel_id + AND relExp.exp_id = rawExp.exp_id + AND relExp.cam_id = camRun.cam_id Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_deleterelstack.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_deleterelstack.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_deleterelstack.sql (revision 37823) @@ -0,0 +1,3 @@ +DELETE FROM relStack USING relStack, ippRelease, stackRun +WHERE relStack.rel_id = ippRelease.rel_id + AND relStack.stack_id = stackRun.stack_id Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_setrelstackcalibratedfromskycal.sql =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_setrelstackcalibratedfromskycal.sql (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/releasetool_setrelstackcalibratedfromskycal.sql (revision 37823) @@ -0,0 +1,10 @@ +UPDATE relStack + JOIN ippRelease USING(rel_id) + JOIN skycell USING(tess_id, skycell_id) + JOIN stackRun USING(stack_id, tess_id, skycell_id) + JOIN skycalRun USING(stack_id) -- join to skycal using stack_id not skycal_id ... + JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id +SET relStack.skycal_id = skycalRun.skycal_id, -- ... because we set skycal_id here + relStack.zpt_obs = skycalResult.zpt_obs, + relStack.zpt_stdev = skycalResult.zpt_stdev, + relStack.state = 'calibrated' Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/addtool.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/addtool.c (revision 37823) @@ -122,4 +122,7 @@ PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); if (strcmp(stage, "cam")== 0) { @@ -146,5 +149,17 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "skycalRun.reduction", "=="); } - + if (strcmp(stage, "diff")==0) { + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); //define using skycalRun label + pxAddLabelSearchArgs (config, where, "-data_group","diffRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "diffRun.reduction", "=="); + } + if (strcmp(stage, "fullforce")==0) { + pxAddLabelSearchArgs (config, where, "-label", "fullForceRun.label", "=="); //define using skycalRun label + pxAddLabelSearchArgs (config, where, "-data_group","fullForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "fullForceRun.reduction", "=="); + } + + + if (!psListLength(where->list)) { psFree(where); @@ -258,5 +273,37 @@ } - + if (strcmp(stage,"diff") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_diff_id_dvo.sql\n%s\n", dvodb,stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_diff_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_diff_id.sql\n%s\n",stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_diff_id.sql"); + // inherit dvodb from skycalRun, avoid matching NULL + psStringAppend(&dvodb_string, "(diffRun.dvodb IS NOT NULL AND previous_dvodb = diffRun.dvodb)"); + // this is silly, there is no dvodb in skycalRun...? + } + } + + if (strcmp(stage,"fullforce") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_ff_id_dvo.sql\n%s\n", dvodb,stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_ff_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_ff_id.sql\n%s\n",stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_ff_id.sql"); + // inherit dvodb from skycalRun, avoid matching NULL + psStringAppend(&dvodb_string, "(fullForceRun.dvodb IS NOT NULL AND previous_dvodb = fullForceRun.dvodb)"); + // this is silly, there is no dvodb in skycalRun...? + } + } @@ -324,4 +371,15 @@ psStringAppend(&query, " GROUP BY skycal_id, sky_id, stack_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there } + //needs to be checked HAF xxx + if (strcmp(stage,"diff") == 0) { + psStringAppend(&query, " GROUP BY diff_id, diff_skyfile_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there + } + //needs to be checked HAF xxx + if (strcmp(stage,"fullforce") == 0) { + psStringAppend(&query, " GROUP BY ff_id, warp_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there + } + + + psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); @@ -476,4 +534,58 @@ } + if (strcmp(stage,"diff") == 0) { + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + diffRunRow *row = diffRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into diffRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, diff_id %" PRId64, row->label, row->diff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, diff_id %" PRId64, row->label, row->diff_id); + psFree(output); + return false; + } + + psFree(row); + } + } + + if (strcmp(stage,"fullforce") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + fullForceRunRow *row = fullForceRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->ff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->ff_id); + psFree(output); + return false; + } + + psFree(row); + } + } @@ -672,4 +784,106 @@ } + if (strcmp(stage,"diff") == 0) { + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + // psS64 stage_extra1 = 0; + diffRunRow *row = diffRunObjectFromMetadata(md); + stage_id = row->diff_id; + bool status = false; + psS32 stage_extra1 = psMetadataLookupS32(&status, md, "diff_skyfile_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item diff_skyfile_id"); + return false; + } + + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + stage_extra1, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + } + + if (strcmp(stage,"fullforce") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + fullForceRunRow *row = fullForceRunObjectFromMetadata(md); + stage_id = row->ff_id; + bool status = false; + psS32 stage_extra1 = psMetadataLookupS32(&status, md, "warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item warp_id"); + return false; + } + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + stage_extra1, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + } + + + @@ -695,4 +909,7 @@ PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); pxcamGetSearchArgs (config, where); // most search arguments based on camera @@ -709,14 +926,20 @@ psString query = NULL; if (strcmp(stage, "cam")==0) { - query = psStringCopy("UPDATE addRun JOIN camRun on cam_id = stage_id JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + query = psStringCopy("UPDATE addRun JOIN camRun on (cam_id = stage_id and stage = 'cam') JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); } if (strcmp(stage, "stack")==0) { - query = psStringCopy("UPDATE addRun JOIN stackRun on stack_id = stage_id"); + query = psStringCopy("UPDATE addRun JOIN stackRun on (stack_id = stage_id and stage = 'stack')"); } if (strcmp(stage, "staticsky")==0) { - query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); + query = psStringCopy("UPDATE addRun JOIN staticskyRun on (sky_id = stage_id and stage = 'staticsky')"); } if (strcmp(stage, "skycal")==0) { - query = psStringCopy("UPDATE addRun JOIN skycalRun on skycal_id = stage_id"); + query = psStringCopy("UPDATE addRun JOIN skycalRun on (skycal_id = stage_id and stage = 'skycal')"); + } + if (strcmp(stage, "diff")==0) { + query = psStringCopy("UPDATE addRun JOIN diffInputSkyfile on (diff_id = stage_id and diff_skyfile_id = stage_extra1 and stage = 'diff') JOIN diffSkyfile on (diff_id, skycell_id) JOIN diffRun using (diff_id)"); + } + if (strcmp(stage, "fullforce")==0) { + query = psStringCopy("UPDATE addRun JOIN fullForceResult on (ff_id = stage_id and warp_id = stage_extra1 and stage = 'fullforce') JOIN fullForceRun on (ff_id)"); } @@ -743,4 +966,6 @@ PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); pxcamGetSearchArgs (config, where); @@ -764,4 +989,10 @@ query = pxDataGet("addtool_find_pendingexp_skycal.sql"); } + if (strcmp(stage, "diff")==0) { + query = pxDataGet("addtool_find_pendingexp_diff.sql"); + } + if (strcmp(stage, "fullforce")==0) { + query = pxDataGet("addtool_find_pendingexp_ff.sql"); + } @@ -790,4 +1021,15 @@ psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1"); } + if (strcmp(stage, "diff") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "diff_id, stage_extra1"); + } + if (strcmp(stage, "fullforce") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "ff_id, stage_extra1"); + } + + + // treat limit == 0 as "no limit" if (limit) { @@ -994,4 +1236,9 @@ } else if (strcmp (stage,"skycal") == 0) { query = pxDataGet("addtool_find_processedexp_skycal.sql"); + } else if (strcmp (stage,"diff") == 0) { + query = pxDataGet("addtool_find_processedexp_diff.sql"); + } else if (strcmp (stage,"fullforce") == 0) { + query = pxDataGet("addtool_find_processedexp_ff.sql"); + } else { psFree(where); @@ -1112,4 +1359,10 @@ if (strcmp(stage, "skycal") == 0) { query = pxDataGet("addtool_revertprocessedexp_skycal.sql"); + } + if (strcmp(stage, "diff") == 0) { + query = pxDataGet("addtool_revertprocessedexp_diff.sql"); + } + if (strcmp(stage, "fullforce") == 0) { + query = pxDataGet("addtool_revertprocessedexp_ff.sql"); } Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/addtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/addtoolConfig.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/addtoolConfig.c (revision 37823) @@ -55,4 +55,6 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal_id", 0); pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by ff_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by camRun data_group", NULL); @@ -83,4 +85,6 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id",0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id", 0, "search by _ff_id",0); pxcamSetSearchArgs(updaterunArgs); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by addRun label", NULL); @@ -101,7 +105,9 @@ psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack_id", 0); - psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); - psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id",0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-ff_id", 0, "search by ff_id",0); pxcamSetSearchArgs(pendingexpArgs); psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); @@ -236,5 +242,5 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definebyquery", "create runs from cam/skycal/forcedwarp/diff stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-updaterun", "change add run properties", ADDTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-pendingexp", "show pending exps", ADDTOOL_MODE_PENDINGEXP, pendingexpArgs); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/chiptool.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/chiptool.c (revision 37823) @@ -1454,6 +1454,6 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); psString query = pxDataGet("chiptool_run.sql"); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/disttool.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/disttool.c (revision 37823) @@ -605,4 +605,6 @@ PXOPT_COPY_TIME(config->args, where, "-time_stamp_begin", "distRun.time_stamp", ">="); PXOPT_COPY_TIME(config->args, where, "-time_stamp_end", "distRun.time_stamp", "<="); + PXOPT_COPY_S64(config->args, where, "-dist_id_min", "dist_id", ">="); + PXOPT_COPY_S64(config->args, where, "-dist_id_max", "dist_id", "<="); PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/disttoolConfig.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/disttoolConfig.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/disttoolConfig.c (revision 37823) @@ -85,4 +85,6 @@ psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-time_stamp_begin", 0, "limit updates by time_stamp (>=)", NULL); psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-time_stamp_end", 0, "limit updates by time_stamp (<=)", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dist_id_min", 0, "limit updates by dist_id (>=)", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dist_id_max", 0, "limit updates by dist_id (<=)", 0); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL,"-clean", 0, "limit updates to clean distRuns", false); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL,"-full", 0, "limit updates to not clean distRuns", false); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/fftool.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/fftool.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/fftool.c (revision 37823) @@ -859,5 +859,5 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); - PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceSummary.fault", "=="); pxskycellAddWhere(config, where); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptool.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptool.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptool.c (revision 37823) @@ -25,4 +25,5 @@ static bool pendingrunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); +static bool revertsassMode(pxConfig *config); // Exposure level @@ -69,5 +70,6 @@ MODECASE(LAPTOOL_MODE_PENDINGRUN, pendingrunMode); MODECASE(LAPTOOL_MODE_UPDATERUN, updaterunMode); - + MODECASE(LAPTOOL_MODE_REVERTSASS, revertsassMode); + MODECASE(LAPTOOL_MODE_PENDINGEXP, pendingexpMode); MODECASE(LAPTOOL_MODE_EXPOSURES, exposuresMode); @@ -695,4 +697,51 @@ } +static bool revertsassMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sass_id", "final_sass_id", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("laptool_revertsass.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return(false); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\n WHERE %s", whereClause); + psFree(whereClause); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + long numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have updated only one row. updated %ld", numUpdated); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} // Exposure level Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptool.h =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptool.h (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptool.h (revision 37823) @@ -19,4 +19,5 @@ LAPTOOL_MODE_EXPOSURES, LAPTOOL_MODE_STACKS, + LAPTOOL_MODE_REVERTSASS, LAPTOOL_MODE_UPDATEEXP, LAPTOOL_MODE_DIFFCHECK, Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptoolConfig.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/laptoolConfig.c (revision 37823) @@ -117,4 +117,9 @@ ADD_OPT(S64, updaterunArgs, "-set_final_sass_id", "set final stack sass_id", 0); + // -revertsass + psMetadata *revertsassArgs = psMetadataAlloc(); + ADD_OPT(S64, revertsassArgs, "-lap_id", "search by lap run id", 0); + ADD_OPT(S64, revertsassArgs, "-sass_id", "sass id that is being removed", 0); + // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); @@ -136,5 +141,5 @@ ADD_OPT(Bool,stacksArgs, "-simple", "use the simple output format", false); ADD_OPT(U64, stacksArgs, "-limit", "limit result set to N items", 0); - + // -updateexp psMetadata *updateexpArgs = psMetadataAlloc(); @@ -227,4 +232,5 @@ PXOPT_ADD_MODE("-listrun", "", LAPTOOL_MODE_PENDINGRUN, listrunArgs); PXOPT_ADD_MODE("-updaterun", "", LAPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-revertsass", "mode to remove the sass_id", LAPTOOL_MODE_REVERTSASS, revertsassArgs); PXOPT_ADD_MODE("-pendingexp", "", LAPTOOL_MODE_PENDINGEXP, pendingexpArgs); PXOPT_ADD_MODE("-exposures", "", LAPTOOL_MODE_EXPOSURES, exposuresArgs); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/pxadd.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/pxadd.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/pxadd.c (revision 37823) @@ -164,4 +164,14 @@ psMemSetPersistent(query, true); } + else if (strcmp(stage,"diff") == 0) { + query = pxDataGet("addtool_queue_diff_id.sql"); + psMemSetPersistent(query, true); + } + else if (strcmp(stage,"fullforce") == 0) { + query = pxDataGet("addtool_queue_ff_id.sql"); + psMemSetPersistent(query, true); + } + + else { Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetool.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetool.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetool.c (revision 37823) @@ -44,7 +44,10 @@ static bool updaterelexpMode(pxConfig *config); static bool listrelexpMode(pxConfig *config); +static bool deleterelexpMode(pxConfig *config); static bool definerelstackMode(pxConfig *config); static bool updaterelstackMode(pxConfig *config); +static bool setrelstackcalibratedfromskycalMode(pxConfig *config); static bool listrelstackMode(pxConfig *config); +static bool deleterelstackMode(pxConfig *config); static bool summaryMode(pxConfig *config); static bool definerelgroupMode(pxConfig *config); @@ -81,8 +84,11 @@ MODECASE(RELEASETOOL_MODE_UPDATERELEXP, updaterelexpMode); MODECASE(RELEASETOOL_MODE_LISTRELEXP, listrelexpMode); + MODECASE(RELEASETOOL_MODE_DELETERELEXP, deleterelexpMode); MODECASE(RELEASETOOL_MODE_DEFINERELSTACK, definerelstackMode); MODECASE(RELEASETOOL_MODE_UPDATERELSTACK, updaterelstackMode); + MODECASE(RELEASETOOL_MODE_SETRELSTACKCALIBRATEDFROMSKYCAL, setrelstackcalibratedfromskycalMode); MODECASE(RELEASETOOL_MODE_LISTRELSTACK, listrelstackMode); + MODECASE(RELEASETOOL_MODE_DELETERELSTACK, deleterelstackMode); MODECASE(RELEASETOOL_MODE_SUMMARY, summaryMode); @@ -817,4 +823,62 @@ psFree(output); + + return true; +} + +static bool deleterelexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(relexp_id, config->args, "-relexp_id", false, false); + // If relexp_id is supplied that is all that we need + if (!relexp_id) { + // Otherwise we need to identify the release ... + PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false); + PXOPT_LOOKUP_S32(rel_id, config->args, "-rel_id", false, false); + + if (!release_name && !rel_id) { + psError(PXTOOLS_ERR_CONFIG, true, "at least one of -relexp_id, -release_name, or -rel_id is required\n"); + return false; + } + // ... and the exposure + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", false, false); + if (!label && !cam_id && !exp_id && !exp_name) { + psError(PXTOOLS_ERR_CONFIG, true, "at least one of -label, -cam_id, -exp_id, -exp_name is required\n"); + return false; + } + } + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S32(config->args, where, "-relexp_id", "relExp.relexp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "relExp.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "=="); + + psString query = pxDataGet("releasetool_deleterelexp.sql"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("releasetool", PS_LOG_INFO, "Deleted %" PRIu64 " relExp rows", affected); + + + psFree(query); return true; @@ -1028,4 +1092,130 @@ return false; } +static bool setrelstackcalibratedfromskycalMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "at least one of -skycal_label, -skycal_data_group, or -skycal_id is required"); + return false; + } + + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + pxskycellAddWhere(config, where); + + // insure that at least one of these is supplied to select the release + PXOPT_LOOKUP_S64(rel_id, config->args, "-rel_id", false, false); + PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false); + if (!rel_id && !release_name) { + psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -release_name is required"); + return false; + } + + // limit query to the target release. (Note: this will select nothing if release_name and rel_id don't match) + // note: we add these after above to insure that search args are supplied + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "=="); + PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(replace, config->args, "-replace", false); + + psString query = pxDataGet("releasetool_setrelstackcalibratedfromskycal.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (replace) { + // don't cut on relStack.state + psStringAppend(&query, "\nWHERE ");; + } else { + // normal path only update rows in state processed + psStringAppend(&query, "\nWHERE relStack.state = 'processed'\nAND ");; + } + + // Add in the where conditions + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "%s", whereClause); + psFree(whereClause); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " relStack rows", affected); + + return true; +} + +static bool deleterelstackMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(relstack_id, config->args, "-relstack_id", false, false); + // If relexp_id is supplied that is all that we need + if (!relstack_id) { + // Otherwise we need to identify the release ... + PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false); + PXOPT_LOOKUP_S32(rel_id, config->args, "-rel_id", false, false); + + if (!release_name && !rel_id) { + psError(PXTOOLS_ERR_CONFIG, true, "at least one of -relstack_id, -release_name, or -rel_id is required\n"); + return false; + } + // ... and the Stack + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); + if (!label && !stack_id) { + psError(PXTOOLS_ERR_CONFIG, true, "at least one of -label and -stack_id is required\n"); + return false; + } + } + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S32(config->args, where, "-relstack_id", "relStack.relstack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "relStack.stack_id", "=="); + + PXOPT_COPY_S32(config->args, where, "-rel_id", "relStack.rel_id", "=="); + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "=="); + + psString query = pxDataGet("releasetool_deleterelstack.sql"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("releasetool", PS_LOG_INFO, "Deleted %" PRIu64 " relStack rows", affected); + + + psFree(query); + + return true; +} static bool listrelstackMode(pxConfig *config) @@ -1052,5 +1242,5 @@ PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); - PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); + PXOPT_COPY_S32(config->args, where, "-rel_id", "relStack.rel_id", "=="); pxskycellAddWhere(config, where); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetool.h =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetool.h (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetool.h (revision 37823) @@ -34,6 +34,9 @@ RELEASETOOL_MODE_TOCALIBEXP, RELEASETOOL_MODE_LISTRELEXP, + RELEASETOOL_MODE_DELETERELEXP, RELEASETOOL_MODE_DEFINERELSTACK, RELEASETOOL_MODE_UPDATERELSTACK, + RELEASETOOL_MODE_SETRELSTACKCALIBRATEDFROMSKYCAL, + RELEASETOOL_MODE_DELETERELSTACK, RELEASETOOL_MODE_TOCALIBSTACK, RELEASETOOL_MODE_LISTRELSTACK, Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetoolConfig.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetoolConfig.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/releasetoolConfig.c (revision 37823) @@ -67,5 +67,5 @@ psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_ubercal_file", 0, "define ubercal file name", NULL); - // -uptdaterelease + // -updaterelease psMetadata *updatereleaseArgs = psMetadataAlloc(); @@ -183,4 +183,15 @@ psMetadataAddBool(listrelexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -deleterelexp + psMetadata *deleterelexpArgs = psMetadataAlloc(); + psMetadataAddS64(deleterelexpArgs, PS_LIST_TAIL, "-relexp_id", 0, "select by released exposure ID", 0); + psMetadataAddS64(deleterelexpArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exposure ID", 0); + psMetadataAddStr(deleterelexpArgs, PS_LIST_TAIL, "-exp_name", 0, "select by exposure name", NULL); + psMetadataAddS32(deleterelexpArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + psMetadataAddS32(deleterelexpArgs, PS_LIST_TAIL, "-cam_id", 0, "select by cam run ID", 0); + psMetadataAddStr(deleterelexpArgs, PS_LIST_TAIL, "-label", 0, "select by camRun.label", NULL); + psMetadataAddStr(deleterelexpArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL); + + // -definerelstack psMetadata *definerelstackArgs = psMetadataAlloc(); @@ -248,4 +259,38 @@ psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -updaterelstack + psMetadata *updaterelstackArgs = psMetadataAlloc(); + + psMetadata *calibraterelstackArgs = psMetadataAlloc(); + + // set the target release and collection of relStacks (one of these is required) + psMetadataAddStr(calibraterelstackArgs, PS_LIST_TAIL, "-release_name", 0, "define release name", NULL); + psMetadataAddS32(calibraterelstackArgs, PS_LIST_TAIL, "-rel_id", 0, "define release ID", 0); + + // select the processing + psMetadataAddStr(calibraterelstackArgs, PS_LIST_TAIL, "-skycal_label", 0, "select by skycalRun.label", NULL); + psMetadataAddStr(calibraterelstackArgs, PS_LIST_TAIL, "-skycal_data_group", 0, "select by skycalRun.data_group", NULL); + psMetadataAddS64(calibraterelstackArgs, PS_LIST_TAIL, "-skycal_id", 0, "select by skycalRun.skycal_id", 0); + + psMetadataAddStr(calibraterelstackArgs, PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL); + psMetadataAddStr(calibraterelstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id", NULL); + pxskycellAddArguments(calibraterelstackArgs); + + psMetadataAddBool(calibraterelstackArgs, PS_LIST_TAIL, "-replace", 0, "replace any existing calibrations", false); + psMetadataAddU64(calibraterelstackArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -deleterelstack + psMetadata *deleterelstackArgs = psMetadataAlloc(); + psMetadataAddS64(deleterelstackArgs, PS_LIST_TAIL, "-relstack_id", 0, "select by released exposure ID", 0); + psMetadataAddS64(deleterelstackArgs, PS_LIST_TAIL, "-stack_id", 0, "select by stack ID", 0); +// psMetadataAddS64(deleterelstackArgs, PS_LIST_TAIL, "-skycal_id", 0, "select by skycal ID", 0); + psMetadataAddStr(deleterelstackArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL); + psMetadataAddS32(deleterelstackArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + + psMetadataAddStr(deleterelstackArgs, PS_LIST_TAIL, "-label", 0, "select by stackRun.label", NULL); +// psMetadataAddStr(deleterelstackArgs, PS_LIST_TAIL, "-data_group", 0, "select by stackRun.data_group", NULL); +// psMetadataAddStr(deleterelstackArgs, PS_LIST_TAIL, "-skycal_label", 0, "select by skycalRun.label", NULL); +// psMetadataAddStr(deleterelstackArgs, PS_LIST_TAIL, "-skycal_data_group", 0, "select by skycalRun.data_group", NULL); + // -summary psMetadata *summaryArgs = psMetadataAlloc(); @@ -348,7 +393,12 @@ PXOPT_ADD_MODE("-tocalibexp", "list relExps pending calibration", RELEASETOOL_MODE_TOCALIBEXP, tocalibexpArgs); PXOPT_ADD_MODE("-listrelexp", "list released exposures", RELEASETOOL_MODE_LISTRELEXP, listrelexpArgs); + PXOPT_ADD_MODE("-deleterelexp", "delete a released exposure", RELEASETOOL_MODE_DELETERELEXP, deleterelexpArgs); PXOPT_ADD_MODE("-definerelstack", "define a released stack", RELEASETOOL_MODE_DEFINERELSTACK, definerelstackArgs); PXOPT_ADD_MODE("-listrelstack", "list released stacks", RELEASETOOL_MODE_LISTRELSTACK, listrelstackArgs); + PXOPT_ADD_MODE("-updaterelstack", "update parameters of released stacks", RELEASETOOL_MODE_LISTRELSTACK, updaterelstackArgs); + PXOPT_ADD_MODE("-setrelstackcalibratedfromskycal", "update parameters of using skycalResults", RELEASETOOL_MODE_SETRELSTACKCALIBRATEDFROMSKYCAL, calibraterelstackArgs); + PXOPT_ADD_MODE("-deleterelstack", "update parameters of released stacks", RELEASETOOL_MODE_DELETERELSTACK, deleterelstackArgs); + PXOPT_ADD_MODE("-summary", "list stackSummaryes for released stacks", RELEASETOOL_MODE_SUMMARY, summaryArgs); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktool.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktool.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktool.c (revision 37823) @@ -41,4 +41,5 @@ static bool sumskyfileMode(pxConfig *config); static bool sassskyfileMode(pxConfig *config); +static bool updatesassMode(pxConfig *config); static bool revertsumskyfileMode(pxConfig *config); static bool tosummaryMode(pxConfig *config); @@ -82,5 +83,6 @@ MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileMode); MODECASE(STACKTOOL_MODE_SUMSKYFILE, sumskyfileMode); - MODECASE(STACKTOOL_MODE_SASSSKYFILE, sassskyfileMode); + MODECASE(STACKTOOL_MODE_SASSSKYFILE, sassskyfileMode); + MODECASE(STACKTOOL_MODE_UPDATESASS, updatesassMode); MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileMode); MODECASE(STACKTOOL_MODE_TOSUMMARY, tosummaryMode); @@ -756,6 +758,12 @@ PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "=="); PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "=="); - PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S64(sass_id, config->args, "-sass_id", false, false); + + if (sass_id) { + PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); + } + if (!psListLength(where->list)) { psFree(where); @@ -771,5 +779,8 @@ PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); } - + if (sass_id) { + psStringAppend(&query, " JOIN stackAssociationMap USING(stack_id)"); + } + // pxUpdateRun gets parameters from config->args and updates bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true, false); @@ -1295,9 +1306,4 @@ PXOPT_COPY_STR(config->args, where, "-data_group", "stackAssociation.data_group", "LIKE"); -// The following three selectors are incompatible with the sql so omit them -// PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); -// PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); -// PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -1369,4 +1375,49 @@ } +static bool updatesassMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-sass_id", "sass_id", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psMetadata *values = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, values, "-set_data_group", "data_group", "=="); + if (!psListLength(values->list)) { + psFree(values); + psError(PXTOOLS_ERR_CONFIG, false, "data_group to assign not found"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + long rows = psDBUpdateRows(config->dbh, "stackAssociation", where, values); + psFree(values); + psFree(where); + + if (rows != 1) { + psError(PS_ERR_UNKNOWN, false, "should have updated only one row. updated %ld", rows); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} static bool revertsumskyfileMode(pxConfig *config) Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktool.h =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktool.h (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktool.h (revision 37823) @@ -35,4 +35,5 @@ STACKTOOL_MODE_SUMSKYFILE, STACKTOOL_MODE_SASSSKYFILE, + STACKTOOL_MODE_UPDATESASS, STACKTOOL_MODE_REVERTSUMSKYFILE, STACKTOOL_MODE_TOSUMMARY, Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktoolConfig.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/stacktoolConfig.c (revision 37823) @@ -238,4 +238,14 @@ psMetadataAddBool(sassskyfileArgs, PS_LIST_TAIL, "-all", 0, "enable search without arguments", false); + // -updatesass + psMetadata *updatesassArgs = psMetadataAlloc(); + psMetadataAddS64(updatesassArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association id", 0); + psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-data_group", 0, "search by data group", 0); + psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-projection_cell", 0, "search by projection cell", 0); + psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess_id", 0); + psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", 0); + + psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-set_data_group", 0, "data_group to assign", 0); + // -revertsumskyfile psMetadata *revertsumskyfileArgs= psMetadataAlloc(); @@ -348,5 +358,6 @@ PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", STACKTOOL_MODE_DONECLEANUP, donecleanupArgs); - PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); + PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); + PXOPT_ADD_MODE("-updatesass", "update data_group of stackAssociation", STACKTOOL_MODE_UPDATESASS, updatesassArgs); PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs); PXOPT_ADD_MODE("-summary", "show runs that have been summarized", STACKTOOL_MODE_SUMMARY, summaryArgs); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/staticskytool.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/staticskytool.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/staticskytool.c (revision 37823) @@ -1057,4 +1057,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); psString query = pxDataGet("staticskytool_defineskycalrun.sql"); @@ -1084,4 +1085,9 @@ } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } if (!p_psDBRunQueryF(config->dbh, query, labelHook)) { @@ -1171,4 +1177,5 @@ PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "skycalRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "skycalRun.state", "=="); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/staticskytoolConfig.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/staticskytoolConfig.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippTools/src/staticskytoolConfig.c (revision 37823) @@ -181,4 +181,5 @@ psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-check_inputs", 0, "list inputs, do not modify database", false); psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(defineskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -updateskycalrun @@ -188,4 +189,5 @@ psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", 0); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); pxskycellAddArguments(updateskycalrunArgs); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/Makefile.am =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/Makefile.am (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/Makefile.am (revision 37823) @@ -4,4 +4,5 @@ gpc1 \ gpc1-tdi \ + gpc2 \ cfh12k \ megacam \ Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/configure.ac =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/configure.ac (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/configure.ac (revision 37823) @@ -15,4 +15,5 @@ gpc1/Makefile gpc1-tdi/Makefile + gpc2/Makefile cfh12k/Makefile megacam/Makefile Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/dvo.photcodes =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/dvo.photcodes (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/dvo.photcodes (revision 37823) @@ -159,4 +159,11 @@ 2016 WISE_W3 ref 0.000 0.000 0.000 - - 0.0000 0 - 0.080 0.000 5.000 0.000 0x0000 0x0000 0x0000 0x0000 2017 WISE_W4 ref 0.000 0.000 0.000 - - 0.0000 0 - 0.080 0.000 5.000 0.000 0x0000 0x0000 0x0000 0x0000 + + # ICRF QSOs have astrometric error stored in dX,dY as 1/100th mas + 2090 ICRF_S ref 0.000 0.000 0.000 - - 0.0000 0 - 0.000 0.001 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 2091 ICRF_C ref 0.000 0.000 0.000 - - 0.0000 0 - 0.000 0.001 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 2092 ICRF_X ref 0.000 0.000 0.000 - - 0.0000 0 - 0.000 0.001 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 2093 ICRF_U ref 0.000 0.000 0.000 - - 0.0000 0 - 0.000 0.001 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 2094 ICRF_K ref 0.000 0.000 0.000 - - 0.0000 0 - 0.000 0.001 0.000 0.000 0x0000 0x0000 0x0000 0x0000 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 nan 0.000 2.000 0.600 0x0000 0x0000 0x0000 0x0000 Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/format_20100723.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/format_20100723.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/format_20100723.config (revision 37823) @@ -210,4 +210,6 @@ FPA.BURNTOOL.APPLIED STR BTOOLAPP CHIP.VIDEOCELL STR CELLMODE + + FPA.SHUTOUTC STR SHUTOUTC FPA.PON.TIME STR PONTIME Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/ppStack.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/ppStack.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/ppStack.config (revision 37823) @@ -277,4 +277,185 @@ + + +STACK_CNP_PV3_IQ_1DG METADATA ## revised -- based on STACK_THREEPI/STACK_MD_DEEP_1DG -- not complete + OUTPUT.NOCOMP BOOL FALSE + OUTPUT.LOGFLUX BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 + SAFE BOOL F + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + ## also unclear what rej mods needed + ## + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_CNP_PV3_REF_1DG METADATA ## revised CNP PV3 REF stack -- not complete + OUTPUT.NOCOMP BOOL TRUE # unclear if this necessary for CNP REF + OUTPUT.LOGFLUX BOOL FALSE # + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # CNP large FWHM, center/edge vary 1-2 pix? + PSF.INPUT.CLIP.NSIGMA F32 1.0 # sample typically on rising side of distribution (set smaller?) + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + ## extra rej -- turn default for initial test run + #THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + #COMBINE.ITER F32 0.5 # Number of rejection iterations per input + #COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + #COMBINE.SYS F32 0.1 # Relative systematic error in combination + #COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + #IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + #NMINPIX S32 4 # Minimum input per pixel + #MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + ## + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_CNP_PV3_DEEP_1DG METADATA ## -- not complete + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 16.0 # large FWHM images up to 20 -- + PSF.INPUT.CLIP.NSIGMA F32 100.0 # do not want to clip really + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 3 # Minimum input per pixel -- more and get crazy structures.. + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels -- many, dont trust SUSPECT + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 (PV2 bias hack fix) +END + + +STACK_MD_PV3_NIGHT_1DG METADATA ## Revised MD 8 input NIGHT stacks for PV3 -- not complete + OUTPUT.NOCOMP BOOL FALSE + OUTPUT.LOGFLUX BOOL FALSE + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 12.0 # what FWHM does diffim become mostly useless, if any? + PSF.INPUT.CLIP.NSIGMA F32 1000.0 # want to use up to MAX, but sig could possibly be small? + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + ## + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # Amount to set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 4 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_MD_PV3_IQ_1DG METADATA ## revised config for MD PV3 IQ stacks -- not complete + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK ## needed? + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # center/edge can vary 1-2 pix, prefer <6 pix but needs to be <10 pix for edges until code improved.. + PSF.INPUT.CLIP.NSIGMA F32 100.0 # any change to inputs can greatly impact edge filling factor.. + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # Amount to set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 4 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_MD_PV3_REF_1DG METADATA ## revised config for MD PV3 refstacks -- not complete + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK ## needed? + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 7.0 # center/edge can vary 1-2 pix, prefer <6 pix but needs to be <7 pix for edges + PSF.INPUT.CLIP.NSIGMA F32 1.0 # sample typically on rising side of distribution (set even smaller?) + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE # Set the target PSF FWHM as the maximum of accepted input FWHM values. + PSF.TARGET.AS.MAX.EPSILON F32 0.1 # Amount to set the target PSF FWHM larger than the maximum input. Target = eps + max(input) + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 4 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + +STACK_MD_PV3_DEEP_1DG METADATA ## revised config for MD PV3 deepstack -- not complete + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE + STACK.TYPE STR DEEP_STACK + PSF.INPUT.CLIP.SIMPLE BOOL True + PSF.INPUT.MAX F32 10.0 # + PSF.INPUT.CLIP.NSIGMA F32 100.0 + PSF.INPUT.THRESH F32 NAN + PSF.INPUT.ASYMMETRY F32 NAN + MATCH.REJ F32 4.0 # need ot tune? + SAFE BOOL FALSE + PSF.TARGET.AS.MAX BOOL TRUE + PSF.TARGET.AS.MAX.EPSILON F32 0.1 + THRESHOLD.MASK F32 0.1 # Threshold for mask deconvolution (0..1) + COMBINE.ITER F32 0.5 # Number of rejection iterations per input + COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) + COMBINE.SYS F32 0.1 # Relative systematic error in combination + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean + IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image + NMINPIX S32 30 # Minimum input per pixel + MASK.VAL STR SUSPECT,MASK.VALUE,CONV.BAD,GHOST # Mask value of input bad pixels + MASK.BLANKBORDER S32 20 # Mask blank border in final stack output + BSCALEOFFSET BOOL FALSE # MUST be FALSE for PV3 +END + + + # Recipe overrides for STACK STACKPHOT METADATA Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/psastro.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/psastro.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc1/psastro.config (revision 37823) @@ -365,5 +365,6 @@ # PSASTRO.CATDIR STR PS1.REF.20120503 # PSASTRO.CATDIR STR PS1.REF.20120524 - PSASTRO.CATDIR STR PS1.REF.20140605 +# PSASTRO.CATDIR STR PS1.REF.20140605 + PSASTRO.CATDIR STR PS1.REF.20140713 ZERO.POINT.USE.MEAN BOOL TRUE END @@ -396,4 +397,109 @@ PSASTRO.FIELD.PADDING F32 0.5 END + +CNP_ASTRO_PV3 METADATA + PSASTRO.MAX.ALLOWED.FWHM F32 25 # larger number typical >12, so include 2x larger + PSASTRO.CATDIR STR PS1.REF.20140713 + PSASTRO.GRID.MIN.ANGLE F32 -20 + PSASTRO.GRID.MAX.ANGLE F32 +20 + PSASTRO.GRID.DEL.ANGLE F32 0.5 + PSASTRO.FIELD.PADDING F32 0.5 + DVO.GETSTAR.MAX.RHO F32 5000.0 # globally changed to 10k, huge numbers in soln and larger sigma + DVO.GETSTAR.MIN.MAG F32 8.0 # can go brighter in marginal nights (like MD done every night as possible) + PSASTRO.GRID.NREF.MAX S32 1600 # defaults + PSASTRO.GRID.NRAW.MAX S32 800 + PSASTRO.MIN.INST.MAG.RAW F32 -15.0 # min instrumental magnitude for stars accepted for fitting + PSASTRO.MAX.INST.MAG.RAW F32 -8.5 # reduce instrumental faint end a bit (cuts most if rho>5k anyways) + PSASTRO.GRID.MIN.INST.MAG.RAW F32 -15.0 # min instrumental magnitude for stars accepted for grid search + PSASTRO.GRID.MAX.INST.MAG.RAW F32 -8.0 # max instrumental magnitude for stars accepted for grid search + + ZERO.POINT.USE.MEAN BOOL TRUE + ## refcat -- want to be able to further limit for ZP, general settings + ## -- concern w/ new refcat >20 extra sources w/ larger merr can over-dominating scatter in ZP + ## -- CLAMP will limit both astrom and zp -- CNP are shorter exposures + REFSTAR_CLAMP_MAG_MAX F32 20.0 + REFSTAR.ZP.MIN.MAG F32 8.0 + REFSTAR.ZP.MAX.MAG F32 19.0 + ## for ZP need extra restriction by filter to avoid poorer photometry scatter on faint end + ## -- ZP is by exposure so can easily reduce numbers + ## -- cannot restrict on bright end due to possible marginal nights + ## -- set for merr<0.03 in refcat, similar to MD +# PHOTCODE.DATA MULTI UPDATE + PHOTCODE.DATA MULTI + PHOTCODE.DATA METADATA + FILTER STR g + PHOTCODE STR g + #REFSTAR.ZP.MAX.MAG F32 19.0 + REFSTAR.ZP.MAX.MAG F32 18.0 # changed after PV3.20141111 run to match raw merr + END + PHOTCODE.DATA METADATA + FILTER STR r + PHOTCODE STR r + REFSTAR.ZP.MAX.MAG F32 18.0 + END + PHOTCODE.DATA METADATA + FILTER STR i + PHOTCODE STR i + REFSTAR.ZP.MAX.MAG F32 18.0 + END + PHOTCODE.DATA METADATA + FILTER STR z + PHOTCODE STR z + REFSTAR.ZP.MAX.MAG F32 18.0 + END + PHOTCODE.DATA METADATA + FILTER STR y + PHOTCODE STR y + REFSTAR.ZP.MAX.MAG F32 17.0 + END +END + +MD_ASTRO_PV3 METADATA + ZERO.POINT.BY.EXPOSURE BOOL TRUE + ZERO.POINT.USE.MEAN BOOL TRUE + DVO.GETSTAR.MAX.RHO F32 6000.0 # was globally changed to 10k + PSASTRO.GRID.NREF.MAX S32 1600 # max stars for grid search from ref catalog + PSASTRO.GRID.NRAW.MAX S32 800 # max stars for grid search from raw/image + DVO.GETSTAR.MIN.MAG F32 12.0 # leave similar to default, MD in marginal weather and can lose 1-3 mags + DVO.GETSTAR.MIN.MAG.INST F32 -25.0 + PSASTRO.CATDIR STR PS1.REF.20140713 + ## refcat may be okay astrom but not faint phot -- so need to be able to further limit by ZP + ## -- concern w/ new refcat >20 extra sources w/ larger merr over-dominating scatter in ZP + ## -- CLAMP will limit both astrom and zp -- + REFSTAR_CLAMP_MAG_MAX F32 22.0 + REFSTAR.ZP.MIN.MAG F32 12.0 + REFSTAR.ZP.MAX.MAG F32 19.0 + ## for ZP need extra limit by filter to avoid poorer photometry scatter on faint end + ## -- cannot restrict on bright end due to possible marginal nights + ## -- set for merr<0.02-0.03 in refcat, ~similar for each MD field +# PHOTCODE.DATA MULTI UPDATE + PHOTCODE.DATA MULTI + PHOTCODE.DATA METADATA + FILTER STR g + PHOTCODE STR g + REFSTAR.ZP.MAX.MAG F32 19.0 + END + PHOTCODE.DATA METADATA + FILTER STR r + PHOTCODE STR r + REFSTAR.ZP.MAX.MAG F32 19.0 + END + PHOTCODE.DATA METADATA + FILTER STR i + PHOTCODE STR i + REFSTAR.ZP.MAX.MAG F32 18.5 + END + PHOTCODE.DATA METADATA + FILTER STR z + PHOTCODE STR z + REFSTAR.ZP.MAX.MAG F32 18.5 + END + PHOTCODE.DATA METADATA + FILTER STR y + PHOTCODE STR y + REFSTAR.ZP.MAX.MAG F32 17.5 + END +END + TEST_REFCAT METADATA @@ -430,5 +536,6 @@ DVO.GETSTAR.MIN.MAG.INST F32 -15.0 # PSASTRO.CATDIR STR PS1.REF.20120524 - PSASTRO.CATDIR STR PS1.REF.20140605 +# PSASTRO.CATDIR STR PS1.REF.20140605 + PSASTRO.CATDIR STR PS1.REF.20140713 ZERO.POINT.USE.MEAN BOOL TRUE @@ -525,7 +632,7 @@ ## if defined for MD, needs to have MD name in group so shouldn't be modified except for MD work -## -- ad hoc limit for NREF due to issues fainter than 19 mag in all filters and brighter than 17.0 mag (except y ~16.5) in MD -## -- better solution and other issues need to really be dealt with for PV3 -STATICSKY_MD_DEEPCAL METADATA +## -- was ad hoc limit for NREF~150, rho~6000 due to issues fainter than 19 mag in all filters and brighter than 17.0 mag (except y ~16.5) in MD +## -- for PV3 add refcat min/max mag REFSTAR.ZP.MIN.MAG for ZP range only +STATICSKY_MD_PV3_DEEPCAL METADATA PSASTRO.CATDIR STR PS1.REF.20140713 REFSTAR_MASK BOOL FALSE @@ -537,5 +644,5 @@ PSASTRO.FIELD.PADDING F32 0.01 # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels PSASTRO.MAX.NRAW S32 1500 # include more than MAX.NREF to account for extras like false-positives - PSASTRO.MAX.NREF S32 150 # if edge skycell ~10% then try ~50 to match. increase to help odd bulk offsets + PSASTRO.MAX.NREF S32 500 # if edge skycell ~10% then try ~50 to match. increase to help odd bulk offsets PSASTRO.MIN.INST.MAG.RAW F32 -25.0 # PSASTRO.MAX.INST.MAG.RAW F32 -5.0 @@ -544,6 +651,6 @@ DVO.GETSTAR.MAX.RHO F32 6000.0 # N/sqdeg, slightly larger than MAX.NREF DVO.GETSTAR.MIN.MAG.INST F32 -25.0 # set absurd range to cover large range of exptime, then clamp... - DVO.GETSTAR.MIN.MAG F32 17.0 # only used if keyword problem so also using CLAMP_MAG_MIN - REFSTAR_CLAMP_MAG_MIN F32 17.0 # saturation in MD ~14 even in y-band, so toss out brighter in the list. seeing offset in brighter stars + DVO.GETSTAR.MIN.MAG F32 15.0 # only used if keyword problem so also using CLAMP_MAG_MIN + REFSTAR_CLAMP_MAG_MIN F32 15.0 # saturation in MD ~14 even in y-band, so toss out brighter in the list. offset in brighter stars DVO.GETSTAR.FIXED.ZEROPT F32 0.0 # not really needed, exptime overly dominates @@ -557,7 +664,41 @@ PSASTRO.MIN.NSTAR S32 10 # edges may not have many and require special treatment - KH.CORRECT.APPLY BOOL FALSE -END - - - + KH.CORRECT.APPLY BOOL FALSE + + REFSTAR.ZP.MIN.MAG F32 15.0 # general refcat zp min/max setting -- need to use photcode below + REFSTAR.ZP.MAX.MAG F32 19.0 + ## set min to avoid bright turnover and max to limit merr<0.04 + ## -- +# PHOTCODE.DATA MULTI UPDATE + PHOTCODE.DATA MULTI + PHOTCODE.DATA METADATA + FILTER STR g + PHOTCODE STR g + REFSTAR.ZP.MIN.MAG F32 17.0 # + REFSTAR.ZP.MAX.MAG F32 20.0 # + END + PHOTCODE.DATA METADATA + FILTER STR r + PHOTCODE STR r + REFSTAR.ZP.MIN.MAG F32 17.0 # + REFSTAR.ZP.MAX.MAG F32 19.0 # + END + PHOTCODE.DATA METADATA + FILTER STR i + PHOTCODE STR i + REFSTAR.ZP.MIN.MAG F32 17.0 # + REFSTAR.ZP.MAX.MAG F32 19.0 # + END + PHOTCODE.DATA METADATA + FILTER STR z + PHOTCODE STR z + REFSTAR.ZP.MIN.MAG F32 16.0 # + REFSTAR.ZP.MAX.MAG F32 18.0 # + END + PHOTCODE.DATA METADATA + FILTER STR y + PHOTCODE STR y + REFSTAR.ZP.MIN.MAG F32 15.0 # + REFSTAR.ZP.MAX.MAG F32 17.0 # + END +END Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/Makefile.am =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/Makefile.am (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/Makefile.am (revision 37823) @@ -0,0 +1,22 @@ + +installdir = $(datadir)/ippconfig/gpc2 + +install_files = \ + dvo.config \ + camera.config \ + format_raw.config \ + format_mef.config \ + format_20140521.config \ + ppImage.config \ + ppMerge.config \ + psphot.config \ + psastro.config + +install_DATA = $(install_files) + +install-data-hook: + chmod 0755 $(installdir) + +EXTRA_DIST = $(install_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/camera.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/camera.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/camera.config (revision 37823) @@ -0,0 +1,169 @@ +# Camera configuration file for GPC2: describes the camera + +# File formats that we know about +FORMATS METADATA + MEF STR gpc2/format_mef.config + # RELPHOT STR gpc2/format_relphot.config + FORMAT_20140521 STR gpc2/format_20140521.config + FORMAT_RAW STR gpc2/format_raw.config # default config from the lab (missing MJD, and maybe other stuff) +END + + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + XY01 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY02 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY03 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY04 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY05 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY06 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY10 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY11 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY12 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY13 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY14 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY15 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY16 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY17 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY20 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY21 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY22 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY23 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY24 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY25 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY26 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY27 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY30 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY31 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY32 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY33 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY34 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY35 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY36 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY37 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY40 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY41 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY42 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY43 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY44 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY45 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY46 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY47 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY50 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY51 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY52 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY53 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY54 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY55 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY56 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY57 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY60 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY61 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY62 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY63 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY64 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY65 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY66 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY67 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY71 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY72 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY73 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY74 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY75 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 + XY76 STR xy00 xy01 xy02 xy03 xy04 xy05 xy06 xy07 xy10 xy11 xy12 xy13 xy14 xy15 xy16 xy17 xy20 xy21 xy22 xy23 xy24 xy25 xy26 xy27 xy30 xy31 xy32 xy33 xy34 xy35 xy36 xy37 xy40 xy41 xy42 xy43 xy44 xy45 xy46 xy47 xy50 xy51 xy52 xy53 xy54 xy55 xy56 xy57 xy60 xy61 xy62 xy63 xy64 xy65 xy66 xy67 xy70 xy71 xy72 xy73 xy74 xy75 xy76 xy77 +END + +# internal to external filter names +FILTER.ID METADATA + r MULTI + r STR Not available + r STR g.00002 + r STR r.00002 + r STR i.00002 + r STR z.00002 + r STR y.00002 + r STR w.00002 + # the following were needed for detrend.correct because input and output images seem to be inconsistent sometimes: + # g STR g + # r STR r + # i STR i + # z STR z + # y STR y + # w STR w + OPEN STR OPEN +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa + CHIP STR {CHIP.NAME} + CELL STR {CHIP.NAME}:{CELL.NAME} + fpa STR fpa + chip STR {CHIP.NAME} + cell STR {CHIP.NAME}:{CELL.NAME} +END + +DVO.CAMERADIR STR gpc2 # Camera directory for DVO + +# Recipe options +RECIPES METADATA + PPIMAGE STR gpc2/ppImage.config + PPMERGE STR gpc2/ppMerge.config + PSASTRO STR gpc2/psastro.config + PSPHOT STR gpc2/psphot.config +# PSWARP STR gpc2/pswarp.config +# PSVIDEOPHOT STR gpc2/psvideophot.config +# REJECTIONS STR gpc2/rejections.config # rejection recipe +# PPSTACK STR gpc2/ppStack.config +END + +# reduction classes (recipes which are grouped together) +REDUCTION STR recipes/reductionClasses.mdc + +FITSTYPES STR recipes/fitstypes.mdc + +FILERULES STR recipes/filerules-split.mdc + +EXTNAME.RULES METADATA + CMF.HEAD STR {CHIP.NAME}.hdr + CMF.DATA STR {CHIP.NAME}.psf # use .PSF and .EXT? + CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? + CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XGAL STR {CHIP.NAME}.xgal # use .PSF and .EXT? + CMF.DETEFF STR {CHIP.NAME}.deteff + + PSF.HEAD STR {CHIP.NAME}.hdr + PSF.TABLE STR {CHIP.NAME}.psf_model + PSF.RESID STR {CHIP.NAME}.psf_resid +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS + FPA.OBS STR EXPNUM +END + +PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.NAME} # Rule for generating photcode + +BURNTOOL.STATE.GOOD S16 14 # Value for burntool_state with the most recent bt version. +BURNTOOL.STATE.GOOD.UPDATE S16 13 # for upddate processing accept earlier version + +#FOV_REF F32 20930 # Field of view of unvignetted region in FPA pixels. +#FOV_MAX F32 22720 +#NPIX_REF S32 1357799527 +#NPIX_MAX S32 1482790357 +#NPIX_REF S32 1333206539 +#NPIX_MAX S32 1455690816 +#NPIX_INTERCHIP S32 103794483 + +# This sets the maximum size for the fwhm_major. This is roughly 3", and 98.67% of all exposures in the database are less than this. +MAX_ALLOWED_FWHM F32 12 + +# The set of maskbits that should not be set to NAN in "released" images +# This value is equal to CONV.POOR | STARCORE | SPIKE | SUSPECT = 0x5280 +# unfortunately the perl parser won't accept a hex value +#MASK.NO.CENSOR U32 21120 + +METADATA.COMPRESSION STR 7f # compression mode (nM): n = 1-9, M = f (filtered), h (Huffman), R (run-length) Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/dvo.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/dvo.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/dvo.config (revision 37823) @@ -0,0 +1,73 @@ + +# location of DVO database tables +MOSAICNAME GPC1 + +# keywords used by DVO to interpret the headers + +# keyword abstractions: +#DATE-KEYWORD DATE-OBS +#DATE-MODE yyyy-mm-dd +#UT-KEYWORD UTC-OBS +JD-KEYWORD NONE +MJD-KEYWORD MJD-OBS + +# other keyword abstractions +EXPTIME-KEYWORD EXPTIME +AIRMASS-KEYWORD AIRMASS +CCDNUM-KEYWORD EXTNAME +ST-KEYWORD NONE +OBSERVATORY-LATITUDE 20.7070999146 +OBSERVATORY-LONGITUDE 10.4170608521 + +SUBPIX_DATAFILE NONE + +IMAGE-ID-KEYWORD IMAGEID +SOURCE-ID-KEYWORD SOURCEID + +# instrumental magnitude range for calibration mode +CAL_INSTMAG_MAX 0 +CAL_INSTMAG_MIN 0 + +# exclude overscan region from the dB image boundaries +XOVERSCAN 0 +YOVERSCAN 0 + +# only upload stars within region; a value of 0 means ignore the limit +ADDSTAR_XMIN 0 +ADDSTAR_XMAX 0 +ADDSTAR_YMIN 0 +ADDSTAR_YMAX 0 + +# exclude stars with SN > SNLIMIT (ADDSTAR_SNLIMIT overrides old name MIN_SN_FSTAT) +ADDSTAR_SNLIMIT 0 + +# correlation radius (arcseconds) +ADDSTAR_RADIUS 1.0 + +# scaled correlation radius +ADDSTAR_NSIGMA 0 + +IMAGE_SCATTER 0.075 # mark images POOR if stdev(Mcal) > IMAGE_SCATTER +STAR_SCATTER 0.005 # mark stars POOR if stdev(Mrel) > STAR_SCATTER +IMAGE_OFFSET 0.100 # mark images POOR if abs(delta(Mcal)) > IMAGE_OFFSET +STAR_CHISQ 10.0 # mark stars POOR if Xm > STAR_CHISQ +STAR_TOOFEW 2 # mark star FEW if N(good) < STAR_TOOFEW +GRID_TOOFEW 10 # keep grid FROZEN if N(good) < GRID_TOOFEW +IMAGE_TOOFEW 10 # mark image FEW if N(good) < IMAGE_TOOFEW +IMAGE_GOOD_FRACTION 0.05 # mark image FEW if N(good) < IMAGE_GOOD_FRACTION * Nstars + +SCATTER_LIM 15.0 +MAG_LIM 20.0 # select stars brighter than this for relphot analysis +SIGMA_LIM 0.015 # select measurements with dM < SIGMA_LIM for relphot analysis +# INST_MAG_MIN -17 # optional constraints on magnitude ranges +# INST_MAG_MAX -13 # optional constraints on magnitude ranges + +RELPHOT_GRID_BINNING 256 +RELPHOT_GRID_X 4 +RELPHOT_GRID_Y 4 + +PM_DT_MIN 0.25 +PM_TOOFEW 4 +POS_TOOFEW 1 +RELASTRO_SIGMA_LIM 0.01 # select measurements with dM < SIGMA_LIM for relphot analysis +PAR_FACTOR_MIN 0.2 Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_20140521.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_20140521.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_20140521.config (revision 37823) @@ -0,0 +1,530 @@ +# Refurbished GPC1 +# Change effective from 22 January 2010 (TJD=5219): new CCD temperature keywords + +# How to identify this type +RULE METADATA + ORIGIN STR PS2 + TELESCOP STR PS2 + INSTRUME STR gpc2 + CONTROLR STR STARGRASP + NAMPS S32 64 + MJD-OBS F64 55400.0 # OP: >= +END + +# How to read this data +FILE METADATA + PHU STR CHIP # The FITS file represents a single chip + EXTENSIONS STR CELL # The extensions represent cells + FPA.OBS STR FILENAME # A PHU keyword for unique identifier within the hierarchy level + CONTENT STR DETECTOR # How to determine content of FITS file + CONTENT.RULE STR {CHIP.ID} # How to derive the CONTENT when writing +END + +# What's in the FITS file? +CONTENTS METADATA + # CONTENT = chip name : type + CCID71-2-03-1 STR XY02:GPCChip + CCID71-2-07-2 STR XY03:GPCChip + CCID71-3-08-1 STR XY04:GPCChip + CCID71-1-05-1 STR XY05:GPCChip + CCID71-3-11-3 STR XY11:GPCChip + CCID71-1-10-1 STR XY12:GPCChip + CCID71-3-06-3 STR XY13:GPCChip + CCID71-2-03-2 STR XY14:GPCChip + CCID71-2-08-1 STR XY15:GPCChip + CCID71-2-11-1 STR XY16:GPCChip + CCID71-1-08-4 STR XY21:GPCChip + CCID71-3-03-3 STR XY22:GPCChip + CCID71-3-06-1 STR XY23:GPCChip + CCID71-3-06-2 STR XY24:GPCChip + CCID71-3-06-4 STR XY25:GPCChip + CCID71-1-06-1 STR XY26:GPCChip + CCID71-1-08-2 STR XY30:GPCChip + CCID71-1-05-4 STR XY31:GPCChip + CCID71-1-03-3 STR XY32:GPCChip + CCID71-1-03-2 STR XY33:GPCChip + CCID71-1-06-2 STR XY34:GPCChip + CCID71-1-07-4 STR XY35:GPCChip + CCID71-3-11-2 STR XY36:GPCChip + CCID71-2-12-1 STR XY37:GPCChip + CCID71-1-11-3 STR XY40:GPCChip + CCID71-1-11-2 STR XY41:GPCChip + CCID71-1-09-2 STR XY42:GPCChip + CCID71-2-05-4 STR XY43:GPCChip + CCID71-2-06-2 STR XY44:GPCChip + CCID71-3-03-2 STR XY45:GPCChip + CCID71-2-06-1 STR XY46:GPCChip + CCID71-3-02-1 STR XY47:GPCChip + CCID71-2-10-1 STR XY51:GPCChip + CCID71-2-08-2 STR XY52:GPCChip + CCID71-2-10-2 STR XY53:GPCChip + CCID71-2-11-4 STR XY54:GPCChip + CCID71-3-09-2 STR XY55:GPCChip + CCID71-3-09-1 STR XY56:GPCChip + CCID71-2-03-3 STR XY61:GPCChip + CCID71-1-01-2 STR XY62:GPCChip + CCID71-1-10-2 STR XY63:GPCChip + CCID71-1-05-2 STR XY64:GPCChip + CCID71-1-05-3 STR XY65:GPCChip + CCID71-1-01-4 STR XY66:GPCChip + CCID71-1-03-4 STR XY72:GPCChip + CCID71-2-11-2 STR XY73:GPCChip + CCID71-3-08-2 STR XY74:GPCChip + CCID71-3-11-4 STR XY75:GPCChip + +END + +CHIPS METADATA + GPCChip METADATA + # Extension name, cellName:cellType + xy00 STR xy00:GPCCell + xy10 STR xy10:GPCCell + xy20 STR xy20:GPCCell + xy30 STR xy30:GPCCell + xy40 STR xy40:GPCCell + xy50 STR xy50:GPCCell + xy60 STR xy60:GPCCell + xy70 STR xy70:GPCCell + xy01 STR xy01:GPCCell + xy11 STR xy11:GPCCell + xy21 STR xy21:GPCCell + xy31 STR xy31:GPCCell + xy41 STR xy41:GPCCell + xy51 STR xy51:GPCCell + xy61 STR xy61:GPCCell + xy71 STR xy71:GPCCell + xy02 STR xy02:GPCCell + xy12 STR xy12:GPCCell + xy22 STR xy22:GPCCell + xy32 STR xy32:GPCCell + xy42 STR xy42:GPCCell + xy52 STR xy52:GPCCell + xy62 STR xy62:GPCCell + xy72 STR xy72:GPCCell + xy03 STR xy03:GPCCell + xy13 STR xy13:GPCCell + xy23 STR xy23:GPCCell + xy33 STR xy33:GPCCell + xy43 STR xy43:GPCCell + xy53 STR xy53:GPCCell + xy63 STR xy63:GPCCell + xy73 STR xy73:GPCCell + xy04 STR xy04:GPCCell + xy14 STR xy14:GPCCell + xy24 STR xy24:GPCCell + xy34 STR xy34:GPCCell + xy44 STR xy44:GPCCell + xy54 STR xy54:GPCCell + xy64 STR xy64:GPCCell + xy74 STR xy74:GPCCell + xy05 STR xy05:GPCCell + xy15 STR xy15:GPCCell + xy25 STR xy25:GPCCell + xy35 STR xy35:GPCCell + xy45 STR xy45:GPCCell + xy55 STR xy55:GPCCell + xy65 STR xy65:GPCCell + xy75 STR xy75:GPCCell + xy06 STR xy06:GPCCell + xy16 STR xy16:GPCCell + xy26 STR xy26:GPCCell + xy36 STR xy36:GPCCell + xy46 STR xy46:GPCCell + xy56 STR xy56:GPCCell + xy66 STR xy66:GPCCell + xy76 STR xy76:GPCCell + xy07 STR xy07:GPCCell + xy17 STR xy17:GPCCell + xy27 STR xy27:GPCCell + xy37 STR xy37:GPCCell + xy47 STR xy47:GPCCell + xy57 STR xy57:GPCCell + xy67 STR xy67:GPCCell + xy77 STR xy77:GPCCell + END +END + +# Specify the cell data +CELLS METADATA + GPCCell METADATA + CELL.TRIMSEC.SOURCE STR HEADER + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC.SOURCE STR HEADER + CELL.BIASSEC STR BIASSEC + END +END + + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.FILTERID STR FILTERID + FPA.FILTER STR FILTERID + FPA.RA STR RA + FPA.DEC STR DEC + FPA.RADECSYS STR EQUINOX + FPA.OBSTYPE STR OBSTYPE + FPA.OBJECT STR OBJECT + FPA.OBS.MODE STR OBS_MODE + FPA.COMMENT STR CMMTOBS + FPA.AIRMASS STR AIRMASS + FPA.POSANGLE STR POSANGLE + FPA.ROTANGLE STR ROT + FPA.FOCUS STR M2Z + FPA.TIME STR SHUTOUTC + FPA.ALT STR ALT + FPA.AZ STR AZ + FPA.TEMP STR DETTEM + FPA.M1X STR M1X + FPA.M1Y STR M1Y + FPA.M1Z STR M1Z + FPA.M1TIP STR M1TIP + FPA.M1TILT STR M1TILT + FPA.M2X STR M2X + FPA.M2Y STR M2Y + FPA.M2Z STR M2Z + FPA.M2TIP STR M2TIP + FPA.M2TILT STR M2TILT + FPA.ENV.TEMP STR ENVTEM + FPA.ENV.HUMID STR ENVHUM + FPA.ENV.WIND STR ENVWIN + FPA.ENV.DIR STR ENVDIR + + FPA.TELTEMP.M1 STR TELTEMM1 # Primary mirror temps (C) + FPA.TELTEMP.M1CELL STR TELTEMMS # Primary mirror support temps (C) + FPA.TELTEMP.M2 STR TELTEMM2 # Secondary mirror temps (C + FPA.TELTEMP.SPIDER STR TELTEMSP # Spider temperatures (C) + FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C + FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE + + FPA.PON.TIME STR PONTIME + + CHIP.ID STR DETECTOR + CELL.XBIN STR CCDSUM + CELL.YBIN STR CCDSUM + CELL.X0 STR IMNPIX1 + CELL.Y0 STR IMNPIX2 + CELL.XPARITY STR ATM1_1 + CELL.YPARITY STR ATM2_2 + CHIP.TEMP STR CDETTEM + CHIP.TEMPERATURE STR DETTEM + FPA.EXPOSURE STR EXPREQ # Requested exposure time, presumably camera exposure time + CELL.EXPOSURE STR EXPTIME # Exposure time measured by shutter + CELL.DARKTIME STR DARKTIME # Exposure time for camera + CELL.TIME STR SHUTOUTC # Observation time + CELL.GAIN STR GAIN # Cell gain (e/ADU) + CELL.READNOISE STR RDNOISE # Cell read noise (ADU) +# CELL.SATURATION STR MAXLIN # Saturation point (ADU) +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR PS2 + FPA.INSTRUMENT STR GPC2 + FPA.LONGITUDE STR 10:25:01.417 # West Longitude in Hours 10.4170608521 + FPA.LATITUDE STR 20:42:26.21 # North Latitude in Degrees 20.7070999146 + FPA.ELEVATION F32 3048.0 # altitude in meters + FPA.DETECTOR STR GPC2 + FPA.TIMESYS STR UTC + CHIP.XSIZE S32 4846 + CHIP.YSIZE S32 4868 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CHIP.X0.DEPEND STR CHIP.NAME + CHIP.X0 METADATA + XY01 S32 4971 + XY02 S32 4971 + XY03 S32 4971 + XY04 S32 4971 + XY05 S32 4971 + XY06 S32 4971 + XY10 S32 9942 + XY11 S32 9942 + XY12 S32 9942 + XY13 S32 9942 + XY14 S32 9942 + XY15 S32 9942 + XY16 S32 9942 + XY17 S32 9942 + XY20 S32 14913 + XY21 S32 14913 + XY22 S32 14913 + XY23 S32 14913 + XY24 S32 14913 + XY25 S32 14913 + XY26 S32 14913 + XY27 S32 14913 + XY30 S32 19884 + XY31 S32 19884 + XY32 S32 19884 + XY33 S32 19884 # should be 19872 + XY34 S32 19884 + XY35 S32 19884 + XY36 S32 19884 + XY37 S32 19884 + XY40 S32 20041 + XY41 S32 20041 + XY42 S32 20041 + XY43 S32 20041 # should be 20029 + XY44 S32 20041 + XY45 S32 20041 + XY46 S32 20041 + XY47 S32 20041 + XY50 S32 25012 + XY51 S32 25012 + XY52 S32 25012 + XY53 S32 25012 + XY54 S32 25012 + XY55 S32 25012 + XY56 S32 25012 + XY57 S32 25012 + XY60 S32 29983 + XY61 S32 29983 + XY62 S32 29983 + XY63 S32 29983 + XY64 S32 29983 + XY65 S32 29983 + XY66 S32 29983 + XY67 S32 29983 + XY71 S32 34954 + XY72 S32 34954 + XY73 S32 34954 + XY74 S32 34954 + XY75 S32 34954 + XY76 S32 34954 + END + CHIP.Y0.DEPEND STR CHIP.NAME + CHIP.Y0 METADATA + XY01 S32 10286 + XY02 S32 15429 + XY03 S32 20572 + XY04 S32 25715 + XY05 S32 30858 + XY06 S32 36001 + XY10 S32 5143 + XY11 S32 10286 + XY12 S32 15429 + XY13 S32 20572 + XY14 S32 25715 + XY15 S32 30858 + XY16 S32 36001 + XY17 S32 41144 + XY20 S32 5143 + XY21 S32 10286 + XY22 S32 15429 + XY23 S32 20572 + XY24 S32 25715 + XY25 S32 30858 + XY26 S32 36001 + XY27 S32 41144 + XY30 S32 5143 + XY31 S32 10286 + XY32 S32 15429 + XY33 S32 20572 + XY34 S32 25715 # should be 25755 + XY35 S32 30858 + XY36 S32 36001 + XY37 S32 41144 + XY40 S32 306 + XY41 S32 5449 + XY42 S32 10592 + XY43 S32 15735 # should be 15707 + XY44 S32 20878 # should be 20890 + XY45 S32 26021 + XY46 S32 31164 + XY47 S32 36307 + XY50 S32 306 + XY51 S32 5449 + XY52 S32 10592 + XY53 S32 15735 + XY54 S32 20878 + XY55 S32 26021 + XY56 S32 31164 + XY57 S32 36307 + XY60 S32 306 + XY61 S32 5449 + XY62 S32 10592 + XY63 S32 15735 + XY64 S32 20878 + XY65 S32 26021 + XY66 S32 31164 + XY67 S32 36307 + XY71 S32 5449 + XY72 S32 10592 + XY73 S32 15735 + XY74 S32 20878 + XY75 S32 26021 + XY76 S32 31164 + END + CHIP.XPARITY.DEPEND STR CHIP.NAME + CHIP.XPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CHIP.YPARITY.DEPEND STR CHIP.NAME + CHIP.YPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CELL.READDIR S32 1 +# CELL.BAD S32 0 + CELL.BAD S32 -1000 + CELL.XSIZE S32 590 + CELL.YSIZE S32 598 + CELL.TIMESYS STR UTC + CELL.SATURATION F32 60000 # Saturation point (ADU) +END + +# How to translation PS concepts into database lookups +DATABASE METADATA + # this rule is fragile : does not match the camera +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +END + + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR DEGREES + FPA.DEC STR DEGREES + FPA.LONGITUDE STR HOURS + FPA.LATITUDE STR DEGREES + FPA.TIME STR YEAR.FIRST + CELL.TIME STR YEAR.FIRST + CELL.BINNING STR TOGETHER + # CELL.X0 STR FORTRAN + # CELL.Y0 STR FORTRAN + CELL.READNOISE STR ADU +END + +# Pattern correction information: + +# PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 BOOL FALSE +END + +PATTERN.CONTINUITY BOOL TRUE +# PATTERN.CELL BOOL FALSE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# END Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_mef.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_mef.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_mef.config (revision 37823) @@ -0,0 +1,481 @@ +# The raw GPC data comes off the telescope with each of the chips stored in separate files + +# How to identify this type +RULE METADATA +# TELESCOP STR PS1 +# DETECTOR STR GPC1 + CONTROLR STR STARGRASP + PSFORMAT STR MEF +END + +# How to read this data +FILE METADATA + PHU STR FPA # The FITS file represents a complete camera fpa + EXTENSIONS STR CHIP # The extensions represent chips + FPA.OBS STR FILENAME # A PHU keyword for unique identifier within the hierarchy level +END + +# What's in the FITS file? +CONTENTS METADATA + # CONTENT = chip name : type + CCID58-2-15a1 STR XY01:GPCChip + CCID58-2-18a1 STR XY02:GPCChip + CCID58-1-13a1 STR XY03:GPCChip + CCID58-1-09a1 STR XY04:GPCChip + CCID58-2-16b1 STR XY05:GPCChip + CCID58-2-09b1 STR XY06:GPCChip + CCID58-2-22a1 STR XY10:GPCChip + CCID58-2-13a2 STR XY11:GPCChip + CCID58-1-17a2 STR XY12:GPCChip + CCID58-1-14a2 STR XY13:GPCChip + CCID58-1-09a2 STR XY14:GPCChip + CCID58-2-13b1 STR XY15:GPCChip + CCID58-1-09b1 STR XY16:GPCChip + CCID58-2-10b2 STR XY17:GPCChip + CCID58-1-19a1 STR XY20:GPCChip + CCID58-1-04a1 STR XY21:GPCChip + CCID58-2-09a2 STR XY22:GPCChip + CCID58-1-02a1 STR XY23:GPCChip + CCID58-1-04a2 STR XY24:GPCChip + CCID58-1-01b1 STR XY25:GPCChip + CCID58-2-07b2 STR XY26:GPCChip + CCID58-1-18b1 STR XY27:GPCChip + CCID58-1-12a1 STR XY30:GPCChip + CCID58-2-01a1 STR XY31:GPCChip + CCID58-2-01a2 STR XY32:GPCChip + CCID58-1-02a2 STR XY33:GPCChip + CCID58-2-04a1 STR XY34:GPCChip + CCID58-1-01b2 STR XY35:GPCChip + CCID58-1-02b1 STR XY36:GPCChip + CCID58-2-16b2 STR XY37:GPCChip + CCID58-2-18b2 STR XY40:GPCChip + CCID58-1-03b2 STR XY41:GPCChip + CCID58-1-07b1 STR XY42:GPCChip + CCID58-2-11b2 STR XY43:GPCChip + CCID58-1-05a2 STR XY44:GPCChip + CCID58-1-21a2 STR XY45:GPCChip + CCID58-2-07a2 STR XY46:GPCChip + CCID58-1-21a1 STR XY47:GPCChip + CCID58-1-17b1 STR XY50:GPCChip + CCID58-2-09b2 STR XY51:GPCChip + CCID58-1-04b1 STR XY52:GPCChip + CCID58-1-05b2 STR XY53:GPCChip + CCID58-2-09a1 STR XY54:GPCChip + CCID58-2-12a2 STR XY55:GPCChip + CCID58-1-10a2 STR XY56:GPCChip + CCID58-2-12a1 STR XY57:GPCChip + CCID58-2-22b2 STR XY60:GPCChip + CCID58-1-03b1 STR XY61:GPCChip + CCID58-2-23b2 STR XY62:GPCChip + CCID58-1-02b2 STR XY63:GPCChip + CCID58-1-07a1 STR XY64:GPCChip + CCID58-2-17a2 STR XY65:GPCChip + CCID58-2-14a1 STR XY66:GPCChip + CCID58-2-16a2 STR XY67:GPCChip + CCID58-2-04b2 STR XY71:GPCChip + CCID58-1-14b1 STR XY72:GPCChip + CCID58-1-25b1 STR XY73:GPCChip + CCID58-1-18a1 STR XY74:GPCChip + CCID58-2-16a1 STR XY75:GPCChip + CCID58-2-05a1 STR XY76:GPCChip +END + +CHIPS METADATA + GPCChip METADATA + # Extension name, cellName:cellType + xy00 STR xy00:GPCCell + xy10 STR xy10:GPCCell + xy20 STR xy20:GPCCell + xy30 STR xy30:GPCCell + xy40 STR xy40:GPCCell + xy50 STR xy50:GPCCell + xy60 STR xy60:GPCCell + xy70 STR xy70:GPCCell + xy01 STR xy01:GPCCell + xy11 STR xy11:GPCCell + xy21 STR xy21:GPCCell + xy31 STR xy31:GPCCell + xy41 STR xy41:GPCCell + xy51 STR xy51:GPCCell + xy61 STR xy61:GPCCell + xy71 STR xy71:GPCCell + xy02 STR xy02:GPCCell + xy12 STR xy12:GPCCell + xy22 STR xy22:GPCCell + xy32 STR xy32:GPCCell + xy42 STR xy42:GPCCell + xy52 STR xy52:GPCCell + xy62 STR xy62:GPCCell + xy72 STR xy72:GPCCell + xy03 STR xy03:GPCCell + xy13 STR xy13:GPCCell + xy23 STR xy23:GPCCell + xy33 STR xy33:GPCCell + xy43 STR xy43:GPCCell + xy53 STR xy53:GPCCell + xy63 STR xy63:GPCCell + xy73 STR xy73:GPCCell + xy04 STR xy04:GPCCell + xy14 STR xy14:GPCCell + xy24 STR xy24:GPCCell + xy34 STR xy34:GPCCell + xy44 STR xy44:GPCCell + xy54 STR xy54:GPCCell + xy64 STR xy64:GPCCell + xy74 STR xy74:GPCCell + xy05 STR xy05:GPCCell + xy15 STR xy15:GPCCell + xy25 STR xy25:GPCCell + xy35 STR xy35:GPCCell + xy45 STR xy45:GPCCell + xy55 STR xy55:GPCCell + xy65 STR xy65:GPCCell + xy75 STR xy75:GPCCell + xy06 STR xy06:GPCCell + xy16 STR xy16:GPCCell + xy26 STR xy26:GPCCell + xy36 STR xy36:GPCCell + xy46 STR xy46:GPCCell + xy56 STR xy56:GPCCell + xy66 STR xy66:GPCCell + xy76 STR xy76:GPCCell + xy07 STR xy07:GPCCell + xy17 STR xy17:GPCCell + xy27 STR xy27:GPCCell + xy37 STR xy37:GPCCell + xy47 STR xy47:GPCCell + xy57 STR xy57:GPCCell + xy67 STR xy67:GPCCell + xy77 STR xy77:GPCCell + END +END + +# Specify the cell data +CELLS METADATA + GPCCell METADATA + CELL.TRIMSEC.SOURCE STR HEADER + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC.SOURCE STR HEADER + CELL.BIASSEC STR BIASSEC + END +END + + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.FILTERID STR FILTERID + FPA.FILTER STR FILTERID + FPA.RA STR RA + FPA.DEC STR DEC + FPA.RADECSYS STR EQUINOX + FPA.OBSTYPE STR OBSTYPE + FPA.OBJECT STR OBJECT + FPA.AIRMASS STR AIRMASS + FPA.POSANGLE STR POSANGLE + FPA.ROTANGLE STR ROT + FPA.FOCUS STR M2Z + FPA.TIME STR MJD-OBS + FPA.ALT STR ALT + FPA.AZ STR AZ + FPA.TEMP STR DETTEM + FPA.ZP STR MAG_ZP + CHIP.ID STR DETECTOR + CELL.XBIN STR CCDSUM + CELL.YBIN STR CCDSUM + CELL.X0 STR IMNPIX1 + CELL.Y0 STR IMNPIX2 + CELL.XPARITY STR ATM1_1 + CELL.YPARITY STR ATM2_2 + FPA.EXPOSURE STR EXPREQ # Requested exposure time, presumably camera exposure time + CELL.EXPOSURE STR EXPTIME # Exposure time measured by shutter + CELL.DARKTIME STR DARKTIME # Exposure time for camera + CELL.TIME STR MJD-OBS + CELL.GAIN STR GAIN + CELL.READNOISE STR RDNOISE + CELL.BAD STR LOWLEVEL + CELL.SATURATION STR SATLEVEL +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR PS1 + FPA.INSTRUMENT STR GPC1 + FPA.DETECTOR STR GPC1 + FPA.OBS S32 12345 + FPA.TIMESYS STR TAI + CHIP.XSIZE S32 4846 + CHIP.YSIZE S32 4868 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CHIP.X0.DEPEND STR CHIP.NAME + CHIP.X0 METADATA + XY01 S32 4971 + XY02 S32 4971 + XY03 S32 4971 + XY04 S32 4971 + XY05 S32 4971 + XY06 S32 4971 + XY10 S32 9942 + XY11 S32 9942 + XY12 S32 9942 + XY13 S32 9942 + XY14 S32 9942 + XY15 S32 9942 + XY16 S32 9942 + XY17 S32 9942 + XY20 S32 14913 + XY21 S32 14913 + XY22 S32 14913 + XY23 S32 14913 + XY24 S32 14913 + XY25 S32 14913 + XY26 S32 14913 + XY27 S32 14913 + XY30 S32 19884 + XY31 S32 19884 + XY32 S32 19884 + XY33 S32 19884 + XY34 S32 19884 + XY35 S32 19884 + XY36 S32 19884 + XY37 S32 19884 + XY40 S32 20041 + XY41 S32 20041 + XY42 S32 20041 + XY43 S32 20041 + XY44 S32 20041 + XY45 S32 20041 + XY46 S32 20041 + XY47 S32 20041 + XY50 S32 25012 + XY51 S32 25012 + XY52 S32 25012 + XY53 S32 25012 + XY54 S32 25012 + XY55 S32 25012 + XY56 S32 25012 + XY57 S32 25012 + XY60 S32 29983 + XY61 S32 29983 + XY62 S32 29983 + XY63 S32 29983 + XY64 S32 29983 + XY65 S32 29983 + XY66 S32 29983 + XY67 S32 29983 + XY71 S32 34954 + XY72 S32 34954 + XY73 S32 34954 + XY74 S32 34954 + XY75 S32 34954 + XY76 S32 34954 + END + CHIP.Y0.DEPEND STR CHIP.NAME + CHIP.Y0 METADATA + XY01 S32 10286 + XY02 S32 15429 + XY03 S32 20572 + XY04 S32 25715 + XY05 S32 30858 + XY06 S32 36001 + XY10 S32 5143 + XY11 S32 10286 + XY12 S32 15429 + XY13 S32 20572 + XY14 S32 25715 + XY15 S32 30858 + XY16 S32 36001 + XY17 S32 41144 + XY20 S32 5143 + XY21 S32 10286 + XY22 S32 15429 + XY23 S32 20572 + XY24 S32 25715 + XY25 S32 30858 + XY26 S32 36001 + XY27 S32 41144 + XY30 S32 5143 + XY31 S32 10286 + XY32 S32 15429 + XY33 S32 20572 + XY34 S32 25715 + XY35 S32 30858 + XY36 S32 36001 + XY37 S32 41144 + XY40 S32 306 + XY41 S32 5449 + XY42 S32 10592 + XY43 S32 15735 + XY44 S32 20878 + XY45 S32 26021 + XY46 S32 31164 + XY47 S32 36307 + XY50 S32 306 + XY51 S32 5449 + XY52 S32 10592 + XY53 S32 15735 + XY54 S32 20878 + XY55 S32 26021 + XY56 S32 31164 + XY57 S32 36307 + XY60 S32 306 + XY61 S32 5449 + XY62 S32 10592 + XY63 S32 15735 + XY64 S32 20878 + XY65 S32 26021 + XY66 S32 31164 + XY67 S32 36307 + XY71 S32 5449 + XY72 S32 10592 + XY73 S32 15735 + XY74 S32 20878 + XY75 S32 26021 + XY76 S32 31164 + END + CHIP.XPARITY.DEPEND STR CHIP.NAME + CHIP.XPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CHIP.YPARITY.DEPEND STR CHIP.NAME + CHIP.YPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CELL.READDIR S32 1 + CELL.XSIZE S32 590 + CELL.YSIZE S32 598 + CELL.TIMESYS STR TAI +END + +# How to translation PS concepts into database lookups +DATABASE METADATA +END + + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR DEGREES + FPA.DEC STR DEGREES + FPA.TIME STR MJD + CELL.TIME STR MJD + CELL.BINNING STR TOGETHER +END + Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_raw.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_raw.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/format_raw.config (revision 37823) @@ -0,0 +1,528 @@ +# Refurbished GPC1 +# Change effective from 22 January 2010 (TJD=5219): new CCD temperature keywords + +# How to identify this type +RULE METADATA + ORIGIN STR PS2 + TELESCOP STR PS2 + INSTRUME STR gpc2 + CONTROLR STR STARGRASP + NAMPS S32 64 +END + +# How to read this data +FILE METADATA + PHU STR CHIP # The FITS file represents a single chip + EXTENSIONS STR CELL # The extensions represent cells + FPA.OBS STR FILENAME # A PHU keyword for unique identifier within the hierarchy level + CONTENT STR DETECTOR # How to determine content of FITS file + CONTENT.RULE STR {CHIP.ID} # How to derive the CONTENT when writing +END + +# What's in the FITS file? +CONTENTS METADATA + # CONTENT = chip name : type + CCID71-2-03-1 STR XY02:GPCChip + CCID71-2-07-2 STR XY03:GPCChip + CCID71-3-08-1 STR XY04:GPCChip + CCID71-1-05-1 STR XY05:GPCChip + CCID71-3-11-3 STR XY11:GPCChip + CCID71-1-10-1 STR XY12:GPCChip + CCID71-3-06-3 STR XY13:GPCChip + CCID71-2-03-2 STR XY14:GPCChip + CCID71-2-08-1 STR XY15:GPCChip + CCID71-2-11-1 STR XY16:GPCChip + CCID71-1-08-4 STR XY21:GPCChip + CCID71-3-03-3 STR XY22:GPCChip + CCID71-3-06-1 STR XY23:GPCChip + CCID71-3-06-2 STR XY24:GPCChip + CCID71-3-06-4 STR XY25:GPCChip + CCID71-1-06-1 STR XY26:GPCChip + CCID71-1-08-2 STR XY30:GPCChip + CCID71-1-05-4 STR XY31:GPCChip + CCID71-1-03-3 STR XY32:GPCChip + CCID71-1-03-2 STR XY33:GPCChip + CCID71-1-06-2 STR XY34:GPCChip + CCID71-1-07-4 STR XY35:GPCChip + CCID71-3-11-2 STR XY36:GPCChip + CCID71-2-12-1 STR XY37:GPCChip + CCID71-1-11-3 STR XY40:GPCChip + CCID71-1-11-2 STR XY41:GPCChip + CCID71-1-09-2 STR XY42:GPCChip + CCID71-2-05-4 STR XY43:GPCChip + CCID71-2-06-2 STR XY44:GPCChip + CCID71-3-03-2 STR XY45:GPCChip + CCID71-2-06-1 STR XY46:GPCChip + CCID71-3-02-1 STR XY47:GPCChip + CCID71-2-10-1 STR XY51:GPCChip + CCID71-2-08-2 STR XY52:GPCChip + CCID71-2-10-2 STR XY53:GPCChip + CCID71-2-11-4 STR XY54:GPCChip + CCID71-3-09-2 STR XY55:GPCChip + CCID71-3-09-1 STR XY56:GPCChip + CCID71-2-03-3 STR XY61:GPCChip + CCID71-1-01-2 STR XY62:GPCChip + CCID71-1-10-2 STR XY63:GPCChip + CCID71-1-05-2 STR XY64:GPCChip + CCID71-1-05-3 STR XY65:GPCChip + CCID71-1-01-4 STR XY66:GPCChip + CCID71-1-03-4 STR XY72:GPCChip + CCID71-2-11-2 STR XY73:GPCChip + CCID71-3-08-2 STR XY74:GPCChip + CCID71-3-11-4 STR XY75:GPCChip + +END + +CHIPS METADATA + GPCChip METADATA + # Extension name, cellName:cellType + xy00 STR xy00:GPCCell + xy10 STR xy10:GPCCell + xy20 STR xy20:GPCCell + xy30 STR xy30:GPCCell + xy40 STR xy40:GPCCell + xy50 STR xy50:GPCCell + xy60 STR xy60:GPCCell + xy70 STR xy70:GPCCell + xy01 STR xy01:GPCCell + xy11 STR xy11:GPCCell + xy21 STR xy21:GPCCell + xy31 STR xy31:GPCCell + xy41 STR xy41:GPCCell + xy51 STR xy51:GPCCell + xy61 STR xy61:GPCCell + xy71 STR xy71:GPCCell + xy02 STR xy02:GPCCell + xy12 STR xy12:GPCCell + xy22 STR xy22:GPCCell + xy32 STR xy32:GPCCell + xy42 STR xy42:GPCCell + xy52 STR xy52:GPCCell + xy62 STR xy62:GPCCell + xy72 STR xy72:GPCCell + xy03 STR xy03:GPCCell + xy13 STR xy13:GPCCell + xy23 STR xy23:GPCCell + xy33 STR xy33:GPCCell + xy43 STR xy43:GPCCell + xy53 STR xy53:GPCCell + xy63 STR xy63:GPCCell + xy73 STR xy73:GPCCell + xy04 STR xy04:GPCCell + xy14 STR xy14:GPCCell + xy24 STR xy24:GPCCell + xy34 STR xy34:GPCCell + xy44 STR xy44:GPCCell + xy54 STR xy54:GPCCell + xy64 STR xy64:GPCCell + xy74 STR xy74:GPCCell + xy05 STR xy05:GPCCell + xy15 STR xy15:GPCCell + xy25 STR xy25:GPCCell + xy35 STR xy35:GPCCell + xy45 STR xy45:GPCCell + xy55 STR xy55:GPCCell + xy65 STR xy65:GPCCell + xy75 STR xy75:GPCCell + xy06 STR xy06:GPCCell + xy16 STR xy16:GPCCell + xy26 STR xy26:GPCCell + xy36 STR xy36:GPCCell + xy46 STR xy46:GPCCell + xy56 STR xy56:GPCCell + xy66 STR xy66:GPCCell + xy76 STR xy76:GPCCell + xy07 STR xy07:GPCCell + xy17 STR xy17:GPCCell + xy27 STR xy27:GPCCell + xy37 STR xy37:GPCCell + xy47 STR xy47:GPCCell + xy57 STR xy57:GPCCell + xy67 STR xy67:GPCCell + xy77 STR xy77:GPCCell + END +END + +# Specify the cell data +CELLS METADATA + GPCCell METADATA + CELL.TRIMSEC.SOURCE STR HEADER + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC.SOURCE STR HEADER + CELL.BIASSEC STR BIASSEC + END +END + + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.FILTERID STR FILTERID + FPA.FILTER STR FILTERID + FPA.RA STR RA + FPA.DEC STR DEC + FPA.RADECSYS STR EQUINOX + FPA.OBSTYPE STR OBSTYPE + FPA.OBJECT STR OBJECT + FPA.OBS.MODE STR OBS_MODE + FPA.COMMENT STR CMMTOBS + FPA.AIRMASS STR AIRMASS + FPA.POSANGLE STR POSANGLE + FPA.ROTANGLE STR ROT + FPA.FOCUS STR M2Z + FPA.TIME STR SHUTOUTC + FPA.ALT STR ALT + FPA.AZ STR AZ + FPA.TEMP STR DETTEM + FPA.M1X STR M1X + FPA.M1Y STR M1Y + FPA.M1Z STR M1Z + FPA.M1TIP STR M1TIP + FPA.M1TILT STR M1TILT + FPA.M2X STR M2X + FPA.M2Y STR M2Y + FPA.M2Z STR M2Z + FPA.M2TIP STR M2TIP + FPA.M2TILT STR M2TILT + FPA.ENV.TEMP STR ENVTEM + FPA.ENV.HUMID STR ENVHUM + FPA.ENV.WIND STR ENVWIN + FPA.ENV.DIR STR ENVDIR + + FPA.TELTEMP.M1 STR TELTEMM1 # Primary mirror temps (C) + FPA.TELTEMP.M1CELL STR TELTEMMS # Primary mirror support temps (C) + FPA.TELTEMP.M2 STR TELTEMM2 # Secondary mirror temps (C + FPA.TELTEMP.SPIDER STR TELTEMSP # Spider temperatures (C) + FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C + FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE + + FPA.PON.TIME STR PONTIME + + CHIP.ID STR DETECTOR + CELL.XBIN STR CCDSUM + CELL.YBIN STR CCDSUM + CELL.X0 STR IMNPIX1 + CELL.Y0 STR IMNPIX2 + CELL.XPARITY STR ATM1_1 + CELL.YPARITY STR ATM2_2 + CHIP.TEMP STR CDETTEM + CHIP.TEMPERATURE STR DETTEM + FPA.EXPOSURE STR EXPREQ # Requested exposure time, presumably camera exposure time + CELL.EXPOSURE STR EXPTIME # Exposure time measured by shutter + CELL.DARKTIME STR DARKTIME # Exposure time for camera + CELL.TIME STR SHUTOUTC # Observation time + CELL.GAIN STR GAIN # Cell gain (e/ADU) + CELL.READNOISE STR RDNOISE # Cell read noise (ADU) +# CELL.SATURATION STR MAXLIN # Saturation point (ADU) +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR PS2 + FPA.INSTRUMENT STR GPC2 + FPA.LONGITUDE STR 10:25:01.417 # West Longitude in Hours 10.4170608521 + FPA.LATITUDE STR 20:42:26.21 # North Latitude in Degrees 20.7070999146 + FPA.ELEVATION F32 3048.0 # altitude in meters + FPA.DETECTOR STR GPC2 + FPA.TIMESYS STR UTC + CHIP.XSIZE S32 4846 + CHIP.YSIZE S32 4868 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CHIP.X0.DEPEND STR CHIP.NAME + CHIP.X0 METADATA + XY01 S32 4971 + XY02 S32 4971 + XY03 S32 4971 + XY04 S32 4971 + XY05 S32 4971 + XY06 S32 4971 + XY10 S32 9942 + XY11 S32 9942 + XY12 S32 9942 + XY13 S32 9942 + XY14 S32 9942 + XY15 S32 9942 + XY16 S32 9942 + XY17 S32 9942 + XY20 S32 14913 + XY21 S32 14913 + XY22 S32 14913 + XY23 S32 14913 + XY24 S32 14913 + XY25 S32 14913 + XY26 S32 14913 + XY27 S32 14913 + XY30 S32 19884 + XY31 S32 19884 + XY32 S32 19884 + XY33 S32 19884 # should be 19872 + XY34 S32 19884 + XY35 S32 19884 + XY36 S32 19884 + XY37 S32 19884 + XY40 S32 20041 + XY41 S32 20041 + XY42 S32 20041 + XY43 S32 20041 # should be 20029 + XY44 S32 20041 + XY45 S32 20041 + XY46 S32 20041 + XY47 S32 20041 + XY50 S32 25012 + XY51 S32 25012 + XY52 S32 25012 + XY53 S32 25012 + XY54 S32 25012 + XY55 S32 25012 + XY56 S32 25012 + XY57 S32 25012 + XY60 S32 29983 + XY61 S32 29983 + XY62 S32 29983 + XY63 S32 29983 + XY64 S32 29983 + XY65 S32 29983 + XY66 S32 29983 + XY67 S32 29983 + XY71 S32 34954 + XY72 S32 34954 + XY73 S32 34954 + XY74 S32 34954 + XY75 S32 34954 + XY76 S32 34954 + END + CHIP.Y0.DEPEND STR CHIP.NAME + CHIP.Y0 METADATA + XY01 S32 10286 + XY02 S32 15429 + XY03 S32 20572 + XY04 S32 25715 + XY05 S32 30858 + XY06 S32 36001 + XY10 S32 5143 + XY11 S32 10286 + XY12 S32 15429 + XY13 S32 20572 + XY14 S32 25715 + XY15 S32 30858 + XY16 S32 36001 + XY17 S32 41144 + XY20 S32 5143 + XY21 S32 10286 + XY22 S32 15429 + XY23 S32 20572 + XY24 S32 25715 + XY25 S32 30858 + XY26 S32 36001 + XY27 S32 41144 + XY30 S32 5143 + XY31 S32 10286 + XY32 S32 15429 + XY33 S32 20572 + XY34 S32 25715 # should be 25755 + XY35 S32 30858 + XY36 S32 36001 + XY37 S32 41144 + XY40 S32 306 + XY41 S32 5449 + XY42 S32 10592 + XY43 S32 15735 # should be 15707 + XY44 S32 20878 # should be 20890 + XY45 S32 26021 + XY46 S32 31164 + XY47 S32 36307 + XY50 S32 306 + XY51 S32 5449 + XY52 S32 10592 + XY53 S32 15735 + XY54 S32 20878 + XY55 S32 26021 + XY56 S32 31164 + XY57 S32 36307 + XY60 S32 306 + XY61 S32 5449 + XY62 S32 10592 + XY63 S32 15735 + XY64 S32 20878 + XY65 S32 26021 + XY66 S32 31164 + XY67 S32 36307 + XY71 S32 5449 + XY72 S32 10592 + XY73 S32 15735 + XY74 S32 20878 + XY75 S32 26021 + XY76 S32 31164 + END + CHIP.XPARITY.DEPEND STR CHIP.NAME + CHIP.XPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CHIP.YPARITY.DEPEND STR CHIP.NAME + CHIP.YPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CELL.READDIR S32 1 + CELL.BAD S32 0 + CELL.XSIZE S32 590 + CELL.YSIZE S32 598 + CELL.TIMESYS STR UTC + CELL.SATURATION F32 60000 # Saturation point (ADU) +END + +# How to translation PS concepts into database lookups +DATABASE METADATA + # this rule is fragile : does not match the camera +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +END + + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR DEGREES + FPA.DEC STR DEGREES + FPA.LONGITUDE STR HOURS + FPA.LATITUDE STR DEGREES + FPA.TIME STR YEAR.FIRST + CELL.TIME STR YEAR.FIRST + CELL.BINNING STR TOGETHER + # CELL.X0 STR FORTRAN + # CELL.Y0 STR FORTRAN + CELL.READNOISE STR ADU +END + +# Pattern correction information: + +# PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 BOOL FALSE +END + +PATTERN.CONTINUITY BOOL TRUE +# PATTERN.CELL BOOL FALSE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# END Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/ppImage.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/ppImage.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/ppImage.config (revision 37823) @@ -0,0 +1,1796 @@ + +# binned output image options +BIN1.XBIN S32 16 +BIN1.YBIN S32 16 +BIN2.XBIN S32 150 +BIN2.YBIN S32 150 + +OVERSCAN.SINGLE BOOL FALSE # Reduce overscan to a single value? +OVERSCAN.STAT STR MEDIAN +OVERSCAN.BOXCAR S32 3 + +# Normalization class +NORM.CLASS STR CHIP # How to find the per-class normalizations + +NONLIN BOOL FALSE # apply non-linearity correction +OLDDARK BOOL FALSE + +GAIN.OVERRIDE BOOL TRUE # Override a non-finite gain? + +# use the deburned image instead of the raw, if it exists +USE.DEBURNED.IMAGE BOOL FALSE # use burntool-repaired image? +USE.BEST.BURNTOOL BOOL FALSE # require the best burntooled data +MASK.BURNTOOL BOOL FALSE # Mask potential burntool trails +TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool +NOISEMAP BOOL FALSE # Apply read noise map + +APPLY.BURNTOOL BOOL FALSE # apply burntool correction in ppImage + +# Final detrending options +HAS.VIDEO BOOL FALSE # Treat this OTA as if it has video data +# USE.VIDEO.DARK BOOL TRUE # Use a video dark if we have video data? +USE.VIDEO.MASK BOOL FALSE # Use a video mask if we have video data? + +# aditional auxiliary mask applied to corners of all chips in video cells. +# Is only applied if MASK.AUXMASK and USE.VIDEO.MASK are both true +# AUX.VIDEO.MASK STR neb://any/gpc1/auxmask/mpg.20130930/mask_video_cell_corners.fits + + +TILTYSTREAK.BY.CLASS METADATA # apply the 'tiltystreak' tool +APPLY.XY01 BOOL FALSE +APPLY.XY03 BOOL FALSE +APPLY.XY04 BOOL FALSE +APPLY.XY05 BOOL FALSE +APPLY.XY06 BOOL FALSE +APPLY.XY10 BOOL FALSE +APPLY.XY11 BOOL FALSE +APPLY.XY13 BOOL FALSE +APPLY.XY14 BOOL FALSE +APPLY.XY15 BOOL FALSE +APPLY.XY16 BOOL FALSE +APPLY.XY17 BOOL FALSE +APPLY.XY20 BOOL FALSE +APPLY.XY21 BOOL FALSE +APPLY.XY23 BOOL FALSE +APPLY.XY24 BOOL FALSE +APPLY.XY25 BOOL FALSE +APPLY.XY26 BOOL FALSE +APPLY.XY30 BOOL FALSE +APPLY.XY31 BOOL FALSE +APPLY.XY33 BOOL FALSE +APPLY.XY34 BOOL FALSE +APPLY.XY35 BOOL FALSE +APPLY.XY36 BOOL FALSE +APPLY.XY37 BOOL FALSE +APPLY.XY40 BOOL FALSE +APPLY.XY41 BOOL FALSE +APPLY.XY42 BOOL FALSE +APPLY.XY43 BOOL FALSE +APPLY.XY44 BOOL FALSE +APPLY.XY46 BOOL FALSE +APPLY.XY50 BOOL FALSE +APPLY.XY52 BOOL FALSE +APPLY.XY53 BOOL FALSE +APPLY.XY54 BOOL FALSE +APPLY.XY56 BOOL FALSE +APPLY.XY60 BOOL FALSE +APPLY.XY62 BOOL FALSE +APPLY.XY63 BOOL FALSE +APPLY.XY64 BOOL FALSE +APPLY.XY66 BOOL FALSE +APPLY.XY71 BOOL FALSE +APPLY.XY72 BOOL FALSE +APPLY.XY73 BOOL FALSE +APPLY.XY74 BOOL FALSE +APPLY.XY76 BOOL FALSE + +APPLY.XY02 BOOL TRUE +APPLY.XY12 BOOL TRUE +APPLY.XY22 BOOL TRUE +APPLY.XY27 BOOL TRUE +APPLY.XY32 BOOL TRUE +APPLY.XY45 BOOL TRUE +APPLY.XY47 BOOL TRUE +APPLY.XY51 BOOL TRUE +APPLY.XY55 BOOL TRUE +APPLY.XY57 BOOL TRUE +APPLY.XY61 BOOL TRUE +APPLY.XY65 BOOL TRUE +APPLY.XY67 BOOL TRUE +APPLY.XY75 BOOL TRUE + +OPTIONS.XY02 STR mask=0001011100010111000101110001011100010111000101110001011100010111 # cols: 3,5,6,7 +OPTIONS.XY12 STR mask=0000011100000111000001110000011100000111111101111111011111110111 # col 5,6,7 (+ 0,1,2,3 for row > 4) +OPTIONS.XY22 STR mask=0001001100010011000100110001001100010011000100110001001100010011 # cols: 3,6,7 +OPTIONS.XY27 STR none +OPTIONS.XY32 STR mask=0011011100110111001101110011011100110111001101110011011100110111 # cols: 2,3,5,6,7 +OPTIONS.XY45 STR mask=0111111101111111011111111111111111111111111111111111111111111111 # all except 00, 01, 02 +OPTIONS.XY47 STR mask=1111111001111110011111100111111001111110011111100111111001111110 # cols: 2,3,4,5,6,7 (add 00) +OPTIONS.XY51 STR mask=0011101100111011001110110011101100111011001110110011101100111011 # cols: 2,3,4,6,7 +OPTIONS.XY55 STR mask=0000000011111111111111111111111111111111111111111111111111111111 # all except row 0 +OPTIONS.XY57 STR none +OPTIONS.XY61 STR none +OPTIONS.XY65 STR mask=0111011101110111011101110111011101110111011101110111011101110111 # cols: 1,2,3,5,6,7 +OPTIONS.XY67 STR mask=1100111111001111110011111100111100000000000000000000000000000000 # cols: 0,1,4,5,6,7 except rows 4,5,6,7 +OPTIONS.XY75 STR mask=0001111100011111000111110001111100011111000111110001111100011111 # cols: 3,4,5,6,7 + +# OPTIONS.XY02 STR mask=00010111.00010111.00010111.00010111.00010111.00010111.00010111.00010111 # cols: 3,5,6,7 +END + +# crosstalk measurements : only valid for GPC1 for now +CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +CROSSTALK.CORRECT BOOL FALSE # Subtract model background? + +## XXX use these local variations until we are building real detrend images + +DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + ASTROM METADATA + END + NOISEMAP METADATA + END + LINEARITY METADATA + END +END + +# only apply the fringe for these filters +FRINGE.FILTERS MULTI +FRINGE.FILTERS STR y.00000 + +# Standard chip processing (NOTE: when changing this be sure to check whether changes should be applied to +# CHIP_PRESERVE_BG CHIP_AUXMASK or CHIP_PRESERVE_BG_AUXMASK +CHIP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL FALSE + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL FALSE # Mask potential burntool trails + MASK.AUXMASK BOOL FALSE # augment mask with more restrictive auxiliary mask + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL FALSE # Cell pattern correction using edge continuity + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + +CHIP_NOCOMPRESS METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL TRUE + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + MASK.AUXMASK BOOL FALSE # augment mask with more restrictive auxiliary mask + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + NO.COMPRESS BOOL TRUE # do not compress images +END + +# Chip processing for testing pattern correction +CHIP_PATTERNTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + +# Chip processing with DARKTEST +CHIP_DARKTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + + DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + DETTYPE STR DARKTEST + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + ASTROM METADATA + END + NOISEMAP METADATA + END + END +END + +CHIP_MASKTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + NOISEMAP BOOL TRUE + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARKTEST + EXPTIME STR FPA.EXPOSURE + END + MASK METADATA + DETTYPE STR MASKTEST + END + VIDEOMASK METADATA + DETTYPE STR VIDEOMASKTEST + END + FLAT METADATA + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + END +END + +CHIP_NOISEMAP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + NOISEMAP BOOL TRUE + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + + +# Standard chip processing +CHIP_ADDNOISE METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL TRUE # Cell pattern correction + FRINGE BOOL TRUE # Fringe subtraction + ADDNOISE BOOL TRUE # Add noise to mimic shorter exposure + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + +# Standard chip processing with tiltystreak activated +TILTY METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + TILTYSTREAK.APPLY BOOL TRUE # apply the 'tiltystreak' tool + NOISEMAP BOOL TRUE # Apply read noise map +END + +# Standard chip processing with noisemap activated +NOISY METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool + NOISEMAP BOOL TRUE # Apply read noise map +END + +# Standard chip processing for areas with high stellar density +# Turn remnance masking off --- it kills all the detector +CHIP_DENSE_STARS METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? +END + +# Overscan, bias +PPIMAGE_OB METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction ( + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark +PPIMAGE_OBD METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark -- search for DARK_PREMASK +PPIMAGE_DARKMASK_PROCESS METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK_PREMASK + EXPTIME STR FPA.EXPOSURE + END + END +END + + +# Overscan, bias, dark, shutter +PPIMAGE_OBDS METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter +PPIMAGE_FLATPROC_PREMASK METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK + EXPTIME STR FPA.EXPOSURE + END + END +END + +# Overscan, bias, dark, shutter +PPIMAGE_OBDSF METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field -- use FLAT_PREMASK +PPIMAGE_FLATMASK_PROCESS METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + END +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom +PPIMAGE_DET_ONLY METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + FPA1.FITS BOOL FALSE # Save 1st binned fpa image? + FPA2.FITS BOOL FALSE # Save 2nd binned fpa image? + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + MASK METADATA + DETTYPE STR FLATMASK + END + END +END + +# Overscan, bias, dark, shutter, flat-field, fringe +PPIMAGE_OBDSFR METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + BACKGROUND BOOL TRUE # Subtract background? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom +PPIMAGE_OBDSFRP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + BACKGROUND BOOL TRUE # Subtract background? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom +PPIMAGE_OBDSFRA METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + BACKGROUND BOOL TRUE # Subtract background? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom +PPIMAGE_FLATCORR METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + + DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + DETTYPE STR DARK_PREMASK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + ASTROM METADATA + END + END +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom +PPIMAGE_FLATTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + + DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLATTEST + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + ASTROM METADATA + END + END +END + +# Overscan, bias, dark, shutter, flat-field -- use FLAT_PREMASK +PPIMAGE_PREMASK_PROCESS METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + END +END + +# Photometry and astrometry only (for pre-reduced data) +PPIMAGE_PA METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# JPEG images for different types of residual images +# Save JPEG from BIN1 for bias +PPIMAGE_J1_RESID_B METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + # GPC1 has significant dark structures: use a wide greyscale range + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR VALUE + SCALE.MIN F32 -50.0 + SCALE.MAX F32 +50.0 + END +END + +# Save JPEG from BIN2 for bias +PPIMAGE_J2_RESID_B METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + # GPC1 has significant dark structures: use a wide greyscale range + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR VALUE + SCALE.MIN F32 -50.0 + SCALE.MAX F32 +50.0 + END +END + +# Save JPEG from BIN1 for flat +PPIMAGE_J1_RESID_F METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR FRACTION + SCALE.MIN F32 0.98 + SCALE.MAX F32 1.02 + END +END + +# Save JPEG from BIN2 for flat +PPIMAGE_J2_RESID_F METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR FRACTION + SCALE.MIN F32 0.98 + SCALE.MAX F32 1.02 + END +END + +# generate CTE map image +PPIMAGE_CTEMAP METADATA + BASE.FITS BOOL TRUE # Save base image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + CHECK.CTE BOOL TRUE # measure CTE errors? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK_PREMASK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + END +END + +PPIMAGE_NOISEMAP METADATA + BASE.FITS BOOL TRUE # Save base image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + CHECK.CTE BOOL FALSE # measure CTE errors? + CHECK.NOISE BOOL TRUE # measure cell level noise. + +# DETREND.CONSTRAINTS METADATA +# DARK METADATA +# DETTYPE STR DARK_PREMASK +# EXPTIME STR FPA.EXPOSURE +# END +# FLAT METADATA +# DETTYPE STR FLAT_PREMASK +# FILTER STR FPA.FILTERID +# END +# END +END + + +PR_DARKTEST_NOSUB METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + + DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + DETTYPE STR DARKTEST + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + ASTROM METADATA + END + NOISEMAP METADATA + END + END +END + + +PR METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL TRUE # Cell pattern correction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + +PR_NOPATTERN METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + +# Standard chip processing + auxiliary mask +CHIP_AUXMASK METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL TRUE + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + MASK.AUXMASK BOOL TRUE # augment mask with more restrictive auxiliary mask + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + NO.COMPRESS BOOL TRUE # turn off image compression + DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + AUXMASK METADATA + DETTYPE STR AUXMASK_STS + END + ASTROM METADATA + END + NOISEMAP METADATA + END + LINEARITY METADATA + END + END +END + +# Standard chip processing + auxiliary mask with heavy masking +CHIP_AUXMASK_HEAVY METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL TRUE + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + MASK.AUXMASK BOOL TRUE # augment mask with more restrictive auxiliary mask + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + NO.COMPRESS BOOL TRUE # turn off image compression + DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + AUXMASK METADATA + DETTYPE STR AUXMASK_HEAVY + END + ASTROM METADATA + END + NOISEMAP METADATA + END + LINEARITY METADATA + END + END +END + +# Standard chip processing without background subtraction or pattern correction +CHIP_PRESERVE_BG METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL TRUE + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + MASK.AUXMASK BOOL FALSE # augment mask with more restrictive auxiliary mask + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern coorection using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + + +# Standard chip processing with background removal, but then added back in +CHIP_RESTORE_BG METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL FALSE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + + +# Standard chip processing without background subtraction, pattern correction, or photometry bug applying auxiliary mask +CHIP_PRESERVE_BG_AUXMASK METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL TRUE + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + MASK.AUXMASK BOOL TRUE # augment mask with more restrictive auxiliary mask + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL FALSE # Cell pattern coorection using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + NO.COMPRESS BOOL TRUE # turn off image compression + DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + AUXMASK METADATA + DETTYPE STR AUXMASK_HEAVY + END + ASTROM METADATA + END + NOISEMAP METADATA + END + LINEARITY METADATA + END + END +END + + +# Standard chip processing +CHIP_NOPATTERN METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + +# Standard chip processing for old pre-run5 data +CHIP_PATTERN METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + PATTERN.ROW BOOL TRUE # Subtract pattern noise + PATTERN.CELL BOOL FALSE # Subtract pattern noise + + PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction + XY01 BOOL TRUE + XY02 BOOL TRUE + XY03 BOOL TRUE + XY04 BOOL TRUE + XY05 BOOL TRUE + XY06 BOOL TRUE + XY10 BOOL TRUE + XY11 BOOL TRUE + XY12 BOOL TRUE + XY13 BOOL TRUE + XY14 BOOL TRUE + XY15 BOOL TRUE + XY16 BOOL TRUE + XY17 BOOL TRUE + XY20 BOOL TRUE + XY21 BOOL TRUE + XY22 BOOL TRUE + XY23 BOOL TRUE + XY24 BOOL TRUE + XY25 BOOL TRUE + XY26 BOOL TRUE + XY27 BOOL TRUE + XY30 BOOL TRUE + XY31 BOOL TRUE + XY32 BOOL TRUE + XY33 BOOL TRUE + XY34 BOOL TRUE + XY35 BOOL TRUE + XY36 BOOL TRUE + XY37 BOOL TRUE + XY40 BOOL TRUE + XY41 BOOL TRUE + XY42 BOOL TRUE + XY43 BOOL TRUE + XY44 BOOL TRUE + XY45 BOOL TRUE + XY46 BOOL TRUE + XY47 BOOL TRUE + XY50 BOOL TRUE + XY51 BOOL TRUE + XY52 BOOL TRUE + XY53 BOOL TRUE + XY54 BOOL TRUE + XY55 BOOL TRUE + XY56 BOOL TRUE + XY57 BOOL TRUE + XY60 BOOL TRUE + XY61 BOOL TRUE + XY62 BOOL TRUE + XY63 BOOL TRUE + XY64 BOOL TRUE + XY65 BOOL TRUE + XY66 BOOL TRUE + XY67 BOOL TRUE + XY71 BOOL TRUE + XY72 BOOL TRUE + XY73 BOOL TRUE + XY74 BOOL TRUE + XY75 BOOL TRUE + XY76 BOOL TRUE + END +END + +# test recipes +# currently : default CHIP recipe +APTEST_V1 METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL FALSE # Mask potential burntool trails + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + MASK METADATA + DETTYPE STR MASK + END + END +END + +# test recipes +# currently : default CHIP recipe as normal +APTEST_V2 METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL TRUE # Cell pattern correction + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + +# test recipes +# currently : default CHIP recipe +APTEST_V3 METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL TRUE # Cell pattern correction + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL FALSE # Subtract model background? +END + Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/ppMerge.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/ppMerge.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/ppMerge.config (revision 37823) @@ -0,0 +1,46 @@ +# Mask generation --- already included in default, above +PPMERGE_DARKMASK METADATA + ITER S32 3 + MASK.SUSPECT.SIGMA F32 8.0 # Threshold for suspect pixels (sigma) + MASK.BAD F32 0.1 # Threshold for bad pixels + MASK.MODE STR FRACTION # Meaning of threshold : value, sigma + SAMPLE S32 100000 # Sampling factor for measuring the background +END + +PPMERGE_FLATMASK METADATA + ITER S32 3 + MASK.SUSPECT.SIGMA F32 3.0 # Threshold for suspect pixels (sigma) + MASK.BAD F32 0.1 # Threshold for bad pixels (sigma) + MASK.MODE STR FRACTION # Threshold for bad pixels (sigma) + SAMPLE S32 100000 # Sampling factor for measuring the background +END + +ROWS S32 32 # Number of rows to read at once + +FRINGE.NUM S32 1000 # Number of fringe regions per cell +FRINGE.SIZE S32 5 # Half-size of fringe regions +FRINGE.XSMOOTH S32 3 # Number of smoothing regions per cell in x +FRINGE.YSMOOTH S32 3 # Number of smoothing regions per cell in y +FRINGE.SMOOTH BOOL TRUE # Smooth the output image? +FRINGE.SMOOTH.SIGMA F32 2.0 # Sigma of smoothing Gaussian + +# Ordinates for fitting dark current +# example for linear in darktime only +DARK.ORDINATES METADATA + CELL.EXPOSURE S32 1 +END + +## Dark for 2010-07-02 with new temp concept. +## DARK.ORDINATES METADATA +## CELL.EXPOSURE S32 1 +## TEMP METADATA +## ORDER S32 1 +## RULE STR CHIP.TEMPERATURE * CELL.EXPOSURE +## END +## TEMP2 METADATA +## ORDER S32 1 +## RULE STR CHIP.TEMPERATURE * CHIP.TEMPERATURE * CELL.EXPOSURE +## END +## END + + Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/psastro.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/psastro.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/psastro.config (revision 37823) @@ -0,0 +1,509 @@ + +# save matched stars? +PSASTRO.SAVE.REFMATCH BOOL TRUE # save refstar matches as table in output smf file + +# astrometry matching parameters + +# nominal plate scale (units / pixel) +# XXX we are using pixels here because of the limited dynamic range for +# NAXIS1,2 when loaded by addstar. +PSASTRO.PIXEL.SCALE F32 1.0 + +# pmAstromGridAngle +# max grid offset in FP units (microns) +# use plate-scale to make this in pixels? +PSASTRO.GRID.OFFSET F32 1000. +PSASTRO.GRID.SCALE F32 50 +PSASTRO.GRID.NSTAR.MAX S32 800 # deprecated +PSASTRO.GRID.NREF.MAX S32 800 # max stars accepted for grid search from ref catalog +PSASTRO.GRID.NRAW.MAX S32 1200 # max stars accepted for grid search from raw image + +PSASTRO.MAX.NRAW S32 1000 # max stars accepted for fitting (0 for all) +PSASTRO.MAX.NREF S32 5000 # max stars accepted for fitting (0 for all) + +# extra field for ref stars: +PSASTRO.FIELD.PADDING F32 0.25 # pad by this fraction of the full mosaic (may need to be larger for single chips) + +# +PSASTRO.MIN.INST.MAG.RAW F32 -15.0 # min instrumental magnitude for stars accepted for fitting +PSASTRO.MAX.INST.MAG.RAW F32 -8.0 # max instrumental magnitude for stars accepted for fitting +PSASTRO.GRID.MIN.INST.MAG.RAW F32 -15.0 # min instrumental magnitude for stars accepted for grid search +PSASTRO.GRID.MAX.INST.MAG.RAW F32 -8.0 # max instrumental magnitude for stars accepted for grid search + +PSASTRO.GRID.MIN.ANGLE F32 -1.0 # start angle (degrees) +PSASTRO.GRID.MAX.ANGLE F32 +0.5 +PSASTRO.GRID.DEL.ANGLE F32 0.5 + +# +PSASTRO.GRID.MIN.SCALE F32 1.00 +PSASTRO.GRID.MAX.SCALE F32 1.00 +PSASTRO.GRID.DEL.SCALE F32 0.01 + +# single-chip radius match in pixels +PSASTRO.MATCH.RADIUS.N0 F32 90 +PSASTRO.MATCH.RADIUS.N1 F32 60 +PSASTRO.MATCH.RADIUS.N2 F32 30 +PSASTRO.MATCH.RADIUS.N3 F32 20 +PSASTRO.MATCH.RADIUS.N4 F32 20 +PSASTRO.MATCH.RADIUS.N5 F32 10 +PSASTRO.MATCH.RADIUS.N6 F32 10 +PSASTRO.MATCH.RADIUS.N7 F32 5 +PSASTRO.MATCH.FIT.NITER S32 8 + +# for iterations >= N, require a single match per reference +PSASTRO.MATCH.UNIQ.ITER S32 4 + +# XXX Test for taurus +PSASTRO.GRID.MIN.SIGMA F32 1.0 +PSASTRO.IGNORE STR SATURATED,CRLIMIT,DEFECT,BLEND,FAIL + +# these tweak are in FP units (pixels, currently) +PSASTRO.TWEAK.SCALE F32 1 +PSASTRO.TWEAK.RANGE F32 150 +PSASTRO.TWEAK.SMOOTH F32 2 +PSASTRO.TWEAK.NSIGMA F32 3 + +PSASTRO.USE.MODEL BOOL TRUE + +# PSASTRO.FIX.CHIPS BOOL TRUE +PSASTRO.PIXEL.TOLERANCE F32 3.0 +PSASTRO.ANGLE.TOLERANCE F32 1.0 + +# Mosaic Astrometry options +PSASTRO.MOSAIC.MODE BOOL TRUE + +# for iterations >= N, require a single match per reference +PSASTRO.MOSAIC.CHIP.NITER S32 5 # fit clipping iterations (NOTE: last iteration is NITER, not NITER - 1) +PSASTRO.MOSAIC.CHIP.NSIGMA F32 3 # fit clipping sigmas +PSASTRO.MOSAIC.UNIQ.ITER S32 3 # for iterations >= N, require a single match per reference + +PSASTRO.MOSAIC.MAX.ERROR.N0 F32 0.00 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N1 F32 0.00 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N2 F32 0.00 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N3 F32 0.00 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N4 F32 0.00 # max allow error for valid solution (arcsec) +PSASTRO.MOSAIC.MAX.ERROR.N5 F32 5.00 # max allow error for valid solution (arcsec) + +# mosaic radius match in pixels (do these make sense?) +PSASTRO.MOSAIC.RADIUS.N0 F32 5 +PSASTRO.MOSAIC.RADIUS.N1 F32 10 +PSASTRO.MOSAIC.RADIUS.N2 F32 10 +PSASTRO.MOSAIC.RADIUS.N3 F32 5 +PSASTRO.MOSAIC.RADIUS.N4 F32 3 +PSASTRO.MOSAIC.RADIUS.N5 F32 3 + +PSASTRO.MOSAIC.CHIP.ORDER S32 1 # limit chip-fit order to 1 +PSASTRO.MOSAIC.CHIP.ORDER.N0 S32 0 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N1 S32 0 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N2 S32 1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N3 S32 1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N4 S32 1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N5 S32 3 # fit order (-1 means use default) + +# we need to allow a loose fit if we are fitting to 2mass (50mas internal error -> 100mas limit?) +# if we are fitting against quality digital data, we can require tighter constraints + +PSASTRO.MODEL.REF.CHIP STR XY33 + +PSASTRO.MODEL.SET.BORESITE BOOL FALSE +PSASTRO.MODEL.BORESITE.X F32 -62.0 +PSASTRO.MODEL.BORESITE.Y F32 -139.0 +#PSASTRO.MODEL.BORESITE.X F32 -200.0 +#PSASTRO.MODEL.BORESITE.Y F32 -300.0 + +# maximum accepted fwhm_major (moved from camera.config +PSASTRO.MAX.ALLOWED.FWHM F32 12 + +# *** make sure the choice of CATDIR (dvo database) contains the photcodes desired below +# PSASTRO.CATDIR STR SYNTH.GRIZY +# PSASTRO.CATDIR STR PS1.REF.20120503 +PSASTRO.CATDIR STR PS1.REF.20120524 + + +DVO.GETSTAR.PHOTCODE STR i +DVO.GETSTAR.MAX.RHO F32 3000.0 +DVO.GETSTAR.MIN.MAG F32 12.0 +DVO.GETSTAR.MIN.MAG.INST F32 -25.0 + +# rather than mean, use specified EDGE: +ZERO.POINT.USE.MEAN BOOL TRUE +ZERO.POINT.EDGE.FRACTION F32 0.9 +ZERO.POINT.EDGE.SAMPLE S32 25 +ZERO.POINT.EDGE.SAMPLE.FRACTION F32 0.1 + +PHOTCODE.DATA MULTI +PHOTCODE.DATA METADATA + FILTER STR g + ZEROPT F32 24.563 + PHOTCODE STR g + GHOST_MAX_MAG F32 -16.5 +END +PHOTCODE.DATA METADATA + FILTER STR r + ZEROPT F32 24.750 + PHOTCODE STR r + GHOST_MAX_MAG F32 -20.0 +END +PHOTCODE.DATA METADATA + FILTER STR i + ZEROPT F32 24.611 + PHOTCODE STR i + GHOST_MAX_MAG F32 -25.0 +END +PHOTCODE.DATA METADATA + FILTER STR z + ZEROPT F32 24.250 + PHOTCODE STR z + GHOST_MAX_MAG F32 -25.0 +END +PHOTCODE.DATA METADATA + FILTER STR y + ZEROPT F32 23.320 + PHOTCODE STR y + GHOST_MAX_MAG F32 -25.0 +END +PHOTCODE.DATA METADATA + FILTER STR w + ZEROPT F32 26.3 + PHOTCODE STR r + GHOST_MAX_MAG F32 -20.0 +END + +REFSTAR_MASK BOOL FALSE +REFSTAR_MASK_MAX_MAG F32 -15.0 +REFSTAR_MASK_SATSTAR_MAG_MAX F32 -15.0 +REFSTAR_MASK_SATSTAR_MAG_SLOPE F32 10.15 +# CZW: Original 2010-02-08 position angle offset +REFSTAR_MASK_SATSTAR_POS_ZERO F32 -0.798 +# CZW: Updated 2011-02-18 position angle offset. As above, offset 0.5 degree. +# REFSTAR_MASK_SATSTAR_POS_ZERO F32 -0.78927 +#REFSTAR_MASK_SATSTAR_POS_ZERO F32 -0.80237 +#REFSTAR_MASK_SATSTAR_POS_ZERO F32 -0.81109 +# Length = 10**(SLOPE*(MAG_MAX - InstMag)) - OFFSET + +REFSTAR_MASK_SATSPIKE_MAG_SLOPE F32 0.096 +REFSTAR_MASK_SATSPIKE_MAG_MAX F32 7.35 +REFSTAR_MASK_SATSPIKE_OFFSET F32 200.0 +REFSTAR_MASK_SATSPIKE_WIDTH F32 8.00 +# Width control. if L > L0, W = W0 + (L - L0) * slope +REFSTAR_MASK_SATSPIKE_L0 F32 200.0 +REFSTAR_MASK_SATSPIKE_WIDTH_SLOPE F32 0.01 + +# pair 1 (longer) +# REFSTAR_MASK_SATSPIKE_MAG_MAX1 F32 -16.20 + +# pair 2 (shorter) +# REFSTAR_MASK_SATSPIKE_MAG_MAX2 F32 -17.70 + + + + +REFSTAR_MASK_BLEED BOOL FALSE +REFSTAR_MASK_BLEED_MAG_MAX F32 -15.0 +REFSTAR_MASK_BLEED_MAG_SLOPE F32 5.0 + +PSASTRO.EXTRACT METADATA + EXTRACT_MAX_MAG F32 -19.0 + DVO.GETSTAR.MAX.RHO F32 300.0 +END + +# ghost model file is relative to PATH (eg, ippconfig directory) +GHOST_MODEL STR gpc1/ghost.model.mdc +# GHOST_MAX_MAG F32 -20.0 + +GLINT_LENGTH_MAG_SLOPE F32 2500.0 +GLINT_LENGTH_MAG_ZERO F32 -20.0 +GLINT_WIDTH F32 150.0 +REFSTAR_COUNT_GHOSTS BOOL TRUE +REFSTAR_MASK_GHOST BOOL TRUE + +# GHOST_CENTER_X_0 F32 66.2 +# GHOST_CENTER_X_SLOPE F32 -0.02454 +# GHOST_CENTER_Y_0 F32 16.6 +# GHOST_CENTER_Y_SLOPE F32 +0.01565 +# +# GHOST_INNER_MAJOR_0 F32 32.9 +# GHOST_INNER_MAJOR_SLOPE F32 +0.00612 +# GHOST_INNER_MINOR_0 F32 58.7 +# GHOST_INNER_MINOR_SLOPE F32 -0.00258 +# +# GHOST_OUTER_MAJOR_0 F32 90.6 +# GHOST_OUTER_MAJOR_SLOPE F32 +0.01548 +# GHOST_OUTER_MINOR_0 F32 133.9 +# GHOST_OUTER_MINOR_SLOPE F32 -0.00288 + +PR_RECIPE METADATA + REFSTAR_MASK BOOL FALSE +END + +REFSTAR_MASK_GLINTS BOOL TRUE +GLINT_MAX_MAG F32 -21.0 +GLINT.REGION MULTI + +GLINT.REGION METADATA + REGION STR [-38000:-24000,-20000:+20000] + GLINT.TYPE STR LEFT +END + +GLINT.REGION METADATA + REGION STR [+24000:+38000,-20000:+20000] + GLINT.TYPE STR RIGHT +END + +GLINT.REGION METADATA + REGION STR [-20000:+20000,+24000:+38000:] + GLINT.TYPE STR TOP +END + +GLINT.REGION METADATA + REGION STR [-20000:+20000,-38000:-24000] + GLINT.TYPE STR BOTTOM +END + +REFSTAR_MASK_CROSSTALK_MAG_SLOPE F32 3.44 +REFSTAR_MASK_CROSSTALK_MAG_MAX F32 -14.47 +CROSSTALK_MAX_MAG F32 20.0 +CROSSTALK_MASK BOOL TRUE + +PS1_REFERENCE METADATA + PSASTRO.CATDIR STR PS1.GRIZY +# PSASTRO.CATDIR STR PS1.REF.20120325 + ## PHOTCODE.DATA MULTI UPDATE + ## PHOTCODE.DATA METADATA + ## FILTER STR g + ## ZEROPT F32 24.541 + ## PHOTCODE STR g + ## GHOST_MAX_MAG F32 -20.0 + ## END + ## PHOTCODE.DATA METADATA + ## FILTER STR r + ## ZEROPT F32 24.705.1 + ## PHOTCODE STR r + ## GHOST_MAX_MAG F32 -20.0 + ## END + ## PHOTCODE.DATA METADATA + ## FILTER STR i + ## ZEROPT F32 24.580 + ## PHOTCODE STR i + ## GHOST_MAX_MAG F32 -25.0 + ## END + ## PHOTCODE.DATA METADATA + ## FILTER STR z + ## ZEROPT F32 24.201 + ## PHOTCODE STR z + ## GHOST_MAX_MAG F32 -25.0 + ## END + ## PHOTCODE.DATA METADATA + ## FILTER STR y + ## ZEROPT F32 23.260 + ## PHOTCODE STR y + ## GHOST_MAX_MAG F32 -25.0 + ## END + ## PHOTCODE.DATA METADATA + ## FILTER STR w + ## ZEROPT F32 26.0 + ## PHOTCODE STR r + ## GHOST_MAX_MAG F32 -20.0 + ## END +END + +### MOPS wants dynamic masking turned off +MOPS.TEST METADATA + REFSTAR_MASK BOOL FALSE +END + +SAS_REFERENCE METADATA + PSASTRO.CATDIR STR SAS.REF.V1 +END + +# these values worked to get solutions for the STS observations from 2009 & 2010 +# these field has damage from excessive persistence and the impact of burntool +STS_DATASET METADATA + + # need to boost the total number of stars available for the fit: + # allow more stars per chip and boost the density to which we query + # from the reference database. + PSASTRO.MAX.NRAW S32 2000 # + PSASTRO.MAX.NREF S32 8000 # max stars accepted for fitting (0 for all) + # use wider range of magnitudes for grid search + PSASTRO.GRID.MIN.INST.MAG.RAW F32 -15.0 # min instrumental magnitude for stars accepted for fitting + PSASTRO.GRID.MAX.INST.MAG.RAW F32 -11.0 # max instrumental magnitude for stars accepted for fitting + + # reduce the magnitude range to avoid the bright stars have bad astrometry problem + PSASTRO.MIN.INST.MAG.RAW F32 -12.0 # min instrumental magnitude for stars accepted for fitting + PSASTRO.MAX.INST.MAG.RAW F32 -11.0 # min instrumental magnitude for stars accepted for fitting + + DVO.GETSTAR.MIN.MAG.INST F32 -15.0 + DVO.GETSTAR.MAX.RHO F32 60000.0 + + # the crowding is high, so unless we restrict somewhat tightly early + # on, we get very noisy (and wrong) fits. These values are probably + # too restrictive for the general observation: we would risk missing + # matches. + + # single-chip radius match in pixels + PSASTRO.MATCH.RADIUS.N0 F32 10 + PSASTRO.MATCH.RADIUS.N1 F32 5 + PSASTRO.MATCH.RADIUS.N2 F32 5 + PSASTRO.MATCH.RADIUS.N3 F32 5 + PSASTRO.MATCH.RADIUS.N4 F32 3 + PSASTRO.MATCH.RADIUS.N5 F32 3 + PSASTRO.MATCH.RADIUS.N6 F32 5 + PSASTRO.MATCH.RADIUS.N7 F32 5 + PSASTRO.MATCH.FIT.NITER S32 5 + + # CATDIR with PS1 astrometry from PP5 pointing. Only works for that field + # PSASTRO.CATDIR STR STS.PP5.REFCAT +END + +DEFAULT_RECIPE METADATA +# PSASTRO.CATDIR STR PS1.REF.20120503 + PSASTRO.CATDIR STR PS1.REF.20120524 + ZERO.POINT.USE.MEAN BOOL TRUE +END + +CNP_DATASET METADATA + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.MAX.ALLOWED.FWHM F32 25 +END + +LAP_ASTRO METADATA +# PSASTRO.CATDIR STR 3PI.20110505.REFCAT +# PSASTRO.CATDIR STR SYNTH.GRIZY +# PSASTRO.CATDIR STR PS1.REF.20120503 + PSASTRO.CATDIR STR PS1.REF.20120524 + ZERO.POINT.USE.MEAN BOOL TRUE +END + +SYNTH_CAT METADATA + PSASTRO.CATDIR STR SYNTH.GRIZY +END + +POLE_REFCAT METADATA + PSASTRO.CATDIR STR SYNTH.GRIZY + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.GRID.MIN.ANGLE F32 -20 + PSASTRO.GRID.MAX.ANGLE F32 +20 + PSASTRO.GRID.DEL.ANGLE F32 0.5 + PSASTRO.FIELD.PADDING F32 0.5 +END + +TEST_REFCAT METADATA + PSASTRO.CATDIR STR /data/ipp064.0/ipp/ippRefs/catdir.refcat.20120524.v0 +END + +STATICSKY_TESTCAL METADATA + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE + DVO.GETSTAR.FIXED.ZEROPT F32 25.0 + + PSASTRO.CATDIR STR PS1.REF.20120524 + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.MAX.NRAW S32 0 + PSASTRO.GRID.NRAW.MAX S32 50000 + PSASTRO.MIN.INST.MAG.RAW F32 -21.0 + PSASTRO.MAX.INST.MAG.RAW F32 -15.0 + PSASTRO.GRID.MIN.INST.MAG.RAW F32 -21.0 + PSASTRO.GRID.MAX.INST.MAG.RAW F32 -15.0 + DVO.GETSTAR.MIN.MAG.INST F32 -14.0 +END + +STATICSKY_CAL METADATA + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE + DVO.GETSTAR.FIXED.ZEROPT F32 25.0 + DVO.GETSTAR.MIN.MAG.INST F32 -15.0 + PSASTRO.CATDIR STR PS1.REF.20120524 + ZERO.POINT.USE.MEAN BOOL TRUE + + ## for skycell, radius shouldn't need to be as large as 90 pixel default. deeper stacks, crowded fields can be problems + PSASTRO.MATCH.RADIUS.N0 F32 15 + PSASTRO.MATCH.RADIUS.N1 F32 10 + PSASTRO.MATCH.RADIUS.N2 F32 5 + PSASTRO.MATCH.RADIUS.N3 F32 1 + PSASTRO.MATCH.RADIUS.N4 F32 0.5 + PSASTRO.MATCH.FIT.NITER S32 5 + + PSASTRO.MAX.ERROR F32 0.2 # in arcsec units, if larger than 0.2" on skycell then a problem.. + +END + + +# revised setup for MD nightly stack inputs, should work for case of N~2-30 +# - similar to DEEPCAL except for MIN.NSTAR 5 to attemp more edge skycells if poorer +STATICSKY_NIGHTCAL METADATA + PSASTRO.CATDIR STR PS1.REF.20120524 # reset here to ensure same catalog is used when reprocessing set uniformly + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.FIELD.PADDING F32 0.01 # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels + PSASTRO.MAX.NRAW S32 1000 # include more than MAX.NREF to account for extras like false-positives + PSASTRO.MAX.NREF S32 500 # if edge skycell ~5% then want >50 to match (hit ~40% in normal field matches) + PSASTRO.MIN.INST.MAG.RAW F32 -25.0 # + PSASTRO.MAX.INST.MAG.RAW F32 -5.0 + PSASTRO.GRID.SEARCH BOOL FALSE # skycell/stack and astrom well known, when used could cause problems and not help + #PSASTRO.GRID.NRAW.MAX S32 2000 + #PSASTRO.GRID.NREF.MAX S32 2000 + #PSASTRO.GRID.MIN.INST.MAG.RAW F32 -21.0 + #PSASTRO.GRID.MAX.INST.MAG.RAW F32 -14.0 + DVO.GETSTAR.MAX.RHO F32 6000.0 # slightly larger than MAX.NREF + DVO.GETSTAR.MIN.MAG.INST F32 -25.0 # set absurd range to cover large range of exptime, then clamp... + DVO.GETSTAR.MIN.MAG F32 15.0 # only used if keyword problem + REFSTAR_CLAMP_MAG_MIN F32 15.0 # saturation in MD ~14 even in y-band, so toss out brighter in the list + DVO.GETSTAR.FIXED.ZEROPT F32 0.0 # not really needed, exptime overly dominates + + ## for skycell/stack astrom well known, radius shouldn't need to be as large as 90 pixel default. in deeper stacks can be problems + PSASTRO.MATCH.RADIUS.N0 F32 5 + PSASTRO.MATCH.RADIUS.N1 F32 3 + PSASTRO.MATCH.RADIUS.N2 F32 1 + PSASTRO.MATCH.RADIUS.N3 F32 0.5 + PSASTRO.MATCH.FIT.NITER S32 4 + PSASTRO.MAX.ERROR F32 0.2 # in arcsec units, still liberal limit for well known astrometry for skycells + PSASTRO.MIN.NSTAR S32 5 # edges may not have many and require special treatment +END + + +# revised setup for DEEP/MDS stacks with range of inputs, should work for N~2(edges)--500 inputs with large MIN/MAX.INST.MAG.RAW +STATICSKY_DEEPCAL METADATA + PSASTRO.CATDIR STR PS1.REF.20120524 # reset here to ensure same catalog is used when reprocessing set uniformly + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.FIELD.PADDING F32 0.01 # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels + PSASTRO.MAX.NRAW S32 1000 # include more than MAX.NREF to account for extras like false-positives + PSASTRO.MAX.NREF S32 500 # if edge skycell ~10% then try ~50 to match + PSASTRO.MIN.INST.MAG.RAW F32 -25.0 # + PSASTRO.MAX.INST.MAG.RAW F32 -5.0 + PSASTRO.GRID.SEARCH BOOL FALSE # skycell/stack and astrom well known, when used could cause problems and not help + #PSASTRO.GRID.NRAW.MAX S32 2000 + #PSASTRO.GRID.NREF.MAX S32 2000 + #PSASTRO.GRID.MIN.INST.MAG.RAW F32 -21.0 + #PSASTRO.GRID.MAX.INST.MAG.RAW F32 -14.0 + DVO.GETSTAR.MAX.RHO F32 6000.0 # N/sqdeg, slightly larger than MAX.NREF + DVO.GETSTAR.MIN.MAG.INST F32 -25.0 # set absurd range to cover large range of exptime, then clamp... + DVO.GETSTAR.MIN.MAG F32 15.0 # only used if keyword problem so also using CLAMP_MAG_MIN + REFSTAR_CLAMP_MAG_MIN F32 15.0 # saturation in MD ~14 even in y-band, so toss out brighter in the list + DVO.GETSTAR.FIXED.ZEROPT F32 0.0 # not really needed, exptime overly dominates + + ## for skycell/stack astrom well known, radius shouldn't need to be as large as 90 pixel default. in deeper stacks can be problems + PSASTRO.MATCH.RADIUS.N0 F32 5 + PSASTRO.MATCH.RADIUS.N1 F32 3 + PSASTRO.MATCH.RADIUS.N2 F32 1 + PSASTRO.MATCH.RADIUS.N3 F32 0.5 + PSASTRO.MATCH.FIT.NITER S32 4 + PSASTRO.MAX.ERROR F32 0.2 # in arcsec units, still liberal limit for well known astrometry for skycells + PSASTRO.MIN.NSTAR S32 10 # edges may not have many and require special treatment +END + Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/psphot.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/psphot.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/gpc2/psphot.config (revision 37823) @@ -0,0 +1,105 @@ + +BACKGROUND.XBIN S32 400 # size of background superpixels +BACKGROUND.YBIN S32 400 # size of background superpixels +BACKGROUND.XSAMPLE F32 2.0 # size of background superpixels +BACKGROUND.YSAMPLE F32 2.0 # size of background superpixels + +SOFTEN.VARIANCE BOOL FALSE # XXX drop? soften the errors for bright pixels +SOFTEN.VARIANCE.FRACTION F32 0.000 # CZW 2013-06-20 I've disabled this, as I don't think it's needed. + +PEAKS_SMOOTH_SIGMA F32 2.5 # smoothing kernel sigma in pixels + +PSF_CLUMP_NX S32 3 # subdivide image in to NX x NY regions for +PSF_CLUMP_NY S32 3 # selecting PSF stars +PSF_CLUMP_GRID_SCALE F32 2.5 # XXX too large? size of Sx,Sy image pixel + +PSF.RESIDUALS.RADIUS F32 9.0 # keep this pixel if residual is more significant than this +PSF.RESIDUALS.SPATIAL_ORDER S32 1 # fit spatial variations of the residuals at this order (0,1) + +# PSF model parameters : choose the PSF model desired +PSF_MODEL STR PS_MODEL_PS1_V1 +# vector of simga values to use in when setting the moments window (this is shorter for gpc1 than +# the psphot default) +@PSF.SIGMA.VALUES F32 1.0 2.0 3.0 4.5 6.0 9.0 # comment due to ticket #1476 + +PSPHOT.CR.NSIGMA.SOFTEN F32 0.0025 # Softening parameter for weights in CR stat +PSPHOT.CR.MAX.SIZE F32 0.8 # guess for limit on M_minor in pix^2 +PSPHOT.CR.MAX.MAG F32 -8.0 # guess for limit on Kron Mag +PSPHOT.CR.AUTOSCALE BOOL FALSE + +PSF.TREND.NX S32 6 +PSF.TREND.NY S32 6 + +PSF_MAX_NSTARS S32 2000 # limit number of stars used for PSF model + +# rename NONLINEAR_FIT_SN_LIM? +FULL_FIT_SN_LIM F32 25.0 + +APTREND.ORDER.MAX S32 6 + +PSPHOT.CRMASK.APPLY BOOL TRUE # TEST THIS before enabling. + +# Extended source fit parameters +APTEST_V1 METADATA + BREAK_POINT STR ENSEMBLE +END + +DIAGNOSTIC.PLOTS METADATA + IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE + IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 8 + IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 8 +END + +DIFF METADATA + BACKGROUND.XBIN S32 128 # size of background superpixels + BACKGROUND.YBIN S32 128 # size of background superpixels + DIFF_STATS BOOL TRUE # Calculate diff-specific stats + OUTPUT.FORMAT STR PS1_DV3 # Format of output file + + PSPHOT.EXT.NSIGMA.LIMIT F32 0.0 + PSPHOT.EXT.DIFF.ALTERNATE BOOL TRUE + PSPHOT.EXT.DIFF.ALTERNATE.THRESH F32 1.25 + + # XXX enable these when MOPS is ready + # LMM_FIT_CHISQ_CONVERGENCE BOOL FALSE + # LMM_FIT_GAIN_FACTOR_MODE S32 2 + # PSF_FIT_MIN_TOL F32 0.2 + # EXT_FIT_MIN_TOL F32 0.2 + # PSF_FIT_ITER S32 20 + # EXT_FIT_ITER S32 20 + # FOOTPRINT_CULL_NSIGMA_DELTA F32 10 + # FOOTPRINT_CULL_NSIGMA_MIN F32 2 + + PSPHOT.EXT.FIT.ALL.SOURCES BOOL TRUE + PSPHOT.EXT.FIT.ALL.THRESH F32 1000 +# Lower threshold as requested by MOPS + EXTENDED_SOURCE_MODELS METADATA + TYPE EXTENDED_SOURCE_MODEL MODEL SNLIM PSF_CONVOLVED + TRAIL_RAW EXTENDED_SOURCE_MODEL PS_MODEL_TRAIL 4.75 FALSE + END +END + +SSTF_DIFF METADATA + PSPHOT.EXT.NSIGMA.LIMIT F32 0.0 + PSPHOT.EXT.DIFF.ALTERNATE BOOL TRUE + PSPHOT.EXT.DIFF.ALTERNATE.THRESH F32 1.25 + + PSPHOT.EXT.FIT.ALL.SOURCES BOOL TRUE + PSPHOT.EXT.FIT.ALL.THRESH F32 1000 +# Lower threshold as requested by MOPS + EXTENDED_SOURCE_MODELS METADATA + TYPE EXTENDED_SOURCE_MODEL MODEL SNLIM PSF_CONVOLVED + TRAIL_RAW EXTENDED_SOURCE_MODEL PS_MODEL_TRAIL 4.75 FALSE + END +END + +# Recipe overrides for PR reduction +PR METADATA + BACKGROUND.XBIN S32 128 # size of background superpixels + BACKGROUND.YBIN S32 128 # size of background superpixels + SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background +END + +CHIP_CONSTANT_WEIGHTPHOT METADATA + CONSTANT_PHOTOMETRIC_WEIGHTS BOOL TRUE +END Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/ppImage.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/ppImage.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/ppImage.config (revision 37823) @@ -643,4 +643,28 @@ CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Dark & Overscan subtraction only +PPIMAGE_OD METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction NONLIN BOOL FALSE # Non-linearity correction; not implemented BIAS BOOL FALSE # Bias subtraction @@ -2528,2 +2552,37 @@ END +# Overscan, bias, dark, shutter, flat-field -- use FLAT_PREMASK +PPIMAGE_PREMASK_PROCESS METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK_PREMASK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + END +END + Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/ppStack.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/ppStack.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/ppStack.config (revision 37823) @@ -169,4 +169,5 @@ STACK_THREEPI_ALT METADATA END + STACK_CNP METADATA END @@ -199,4 +200,28 @@ STACK_MDTHREEPI_DEEP_1DG METADATA END + +STACK_MD_PV3_IQ_1DG METADATA +END + +STACK_MD_PV3_REF_1DG METADATA +END + +STACK_MD_PV3_DEEP_1DG METADATA +END + +STACK_MD_PV3_NIGHT_1DG METADATA + STACK.TYPE STR NIGHTLY_STACK +END + +STACK_CNP_PV3_IQ_1DG METADATA +END + +STACK_CNP_PV3_REF_1DG METADATA +END + +STACK_CNP_PV3_DEEP_1DG METADATA +END + + Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/psastro.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/psastro.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/psastro.config (revision 37823) @@ -170,5 +170,12 @@ # report and reject stars with reported magnitudes less than this value. These non physical # magnitudes are errors in the reference catalog (this value is sky magnitude, not instrumental) +# -- and now also for greater than a sky magnitude (should be 24-25?) REFSTAR_CLAMP_MAG_MIN F32 -3.0 +REFSTAR_CLAMP_MAG_MAX F32 30.0 + +# restrict ref catalog for zero point determination +# -- can be put into photcode groups as well +REFSTAR.ZP.MAX.MAG F32 30.0 +REFSTAR.ZP.MIN.MAG F32 -3.0 EXTRACT_MAX_MAG F32 -15.0 @@ -274,4 +281,10 @@ END +CNP_ASTRO_PV3 METADATA +END + +MD_ASTRO_PV3 METADATA +END + SYNTH_CAT METADATA END @@ -299,2 +312,4 @@ END +STATICSKY_MD_PV3_DEEPCAL METADATA +END Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/psphot.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/psphot.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/psphot.config (revision 37823) @@ -233,7 +233,8 @@ # Extended source fit parameters # these models are used for high-quality shape analysis after all sources have been removed -EXTENDED_SOURCE_FITS BOOL FALSE # perform the detailed extended sourc fits? -EXTENDED_SOURCE_FITS_INTERACTIVE BOOL FALSE # perform the detailed extended sourc fits? +EXTENDED_SOURCE_FITS BOOL FALSE # perform the detailed extended source fits? +EXTENDED_SOURCE_FITS_INTERACTIVE BOOL FALSE # perform the detailed extended source fits? EXTENDED_SOURCE_FITS_POISSON BOOL FALSE # use poisson errors for extended source models +EXTENDED_SOURCE_GALAXY_PARAMS BOOL FALSE # measure galaxy parameters EXTENDED_SOURCE_MODELS_SELECTION STR ALL @@ -315,4 +316,5 @@ # ANALYSIS_REGION STR [1000:1600,2800:3400] +GROWTH_FROM_SOURCES BOOL TRUE IGNORE_GROWTH BOOL FALSE CONSTANT_PHOTOMETRIC_WEIGHTS BOOL FALSE @@ -412,5 +414,5 @@ # psphotStack source matching parameters -PSPHOT.STACK.MIN.DETECT.FOR.FORCED S32 2 # number of bands that an object must be detected in before creating +PSPHOT.STACK.MIN.DETECT.FOR.FORCED S32 1 # number of bands that an object must be detected in before creating # sources in undetected bands @@ -426,10 +428,10 @@ PSPHOT.STACK.MATCH.FILTERS METADATA TYPE PSPHOT_STACK.MATCH.FILTER FILTER.ID ORDER MATCH.ALL Y.RATIO.MAX - gband PSPHOT_STACK.MATCH.FILTER g 4 FALSE 0 - rband PSPHOT_STACK.MATCH.FILTER r 2 FALSE 1.7 - iband PSPHOT_STACK.MATCH.FILTER i 1 FALSE 1.6 - zband PSPHOT_STACK.MATCH.FILTER z 3 FALSE 1.65 + gband PSPHOT_STACK.MATCH.FILTER g 4 TRUE 0 + rband PSPHOT_STACK.MATCH.FILTER r 2 TRUE 0 + iband PSPHOT_STACK.MATCH.FILTER i 1 TRUE 0 + zband PSPHOT_STACK.MATCH.FILTER z 3 TRUE 0 yband PSPHOT_STACK.MATCH.FILTER y 5 TRUE 0 - wband PSPHOT_STACK.MATCH.FILTER w 0 FALSE 0 + wband PSPHOT_STACK.MATCH.FILTER w 0 TRUE 0 END @@ -446,5 +448,5 @@ # if true keep all matched sources, even if we get a non-finite aperture flux measurement -PSPHOT.STACK.KEEP.BAD.MATCHES BOOL F +PSPHOT.STACK.KEEP.BAD.MATCHES BOOL T RADIAL_APERTURES BOOL F # calculate flux in circular radial apertures? @@ -479,4 +481,5 @@ STACKPHOT METADATA EXTENDED_SOURCE_FITS BOOL TRUE # perform any of the aperture-like measurements? + EXTENDED_SOURCE_GALAXY_PARAMS BOOL TRUE # measure additional paramters for "galaxies" EXTENDED_SOURCE_SN_LIM F32 10.0 EXTENDED_SOURCE_PETROSIAN BOOL TRUE @@ -494,5 +497,5 @@ RADIAL_APERTURES BOOL T # calculate flux in circular radial apertures? RADIAL_APERTURES_SN_LIM F32 0.0 # S/N limit for radial aperture calculation - OUTPUT.FORMAT STR PS1_SV3 + OUTPUT.FORMAT STR PS1_SV4 # subtract radial profiles for extremely bright stars? @@ -653,5 +656,5 @@ SSTF_DIFF METADATA - PEAKS_NSIGMA_LIMIT_2 F32 4.0 # peak significance threshold + PEAKS_NSIGMA_LIMIT_2 F32 3.5 # peak significance threshold -- MOPS would like to see 4.0->3.5 since ppMops fixed DIFF_STATS BOOL TRUE # Calculate diff-specific stats OUTPUT.FORMAT STR PS1_DV3 # Format of output file Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/reductionClasses.mdc (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/recipes/reductionClasses.mdc (revision 37823) @@ -717,4 +717,83 @@ END +## MD stacks -- PV3 version (NO bscale correction and other changes) +MD_PV3_NIGHT_STACK_1DG METADATA + STACK_PPSTACK STR STACK_MD_PV3_NIGHT_1DG + STACK_PPSUB STR STACK_1DG + STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STATICSKY_CALIBRATION STR STATICSKY_NIGHTCAL +END + +MD_PV3_IQ_STACK_1DG METADATA + STACK_PPSTACK STR STACK_MD_PV3_IQ_1DG + STACK_PPSUB STR STACK_1DG + STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT_1DG + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + STATICSKY_CALIBRATION STR STATICSKY_MD_PV3_DEEPCAL +END + +MD_PV3_REF_STACK_1DG METADATA + STACK_PPSTACK STR STACK_MD_PV3_REF_1DG + STACK_PPSUB STR STACK_1DG + STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT_1DG + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + STATICSKY_CALIBRATION STR STATICSKY_MD_PV3_DEEPCAL +END + +MD_PV3_DEEP_STACK_1DG METADATA + STACK_PPSTACK STR STACK_MD_PV3_DEEP_1DG + STACK_PPSUB STR STACK_1DG + STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT_1DG + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + STATICSKY_CALIBRATION STR STATICSKY_MD_PV3_DEEPCAL +END + +## CNP stacks -- PV3 version (NO bscale correction and other changes) +CNP_PV3_IQ_STACK_1DG METADATA + STACK_PPSTACK STR STACK_CNP_PV3_IQ_1DG + STACK_PPSUB STR STACK_1DG + STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT_1DG + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + STATICSKY_CALIBRATION STR STATICSKY_MD_PV3_DEEPCAL +END + +CNP_PV3_REF_STACK_1DG METADATA + STACK_PPSTACK STR STACK_CNP_PV3_REF_1DG + STACK_PPSUB STR STACK_1DG + STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT_1DG + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + STATICSKY_CALIBRATION STR STATICSKY_MD_PV3_DEEPCAL +END + +CNP_PV3_DEEP_STACK_1DG METADATA + STACK_PPSTACK STR STACK_CNP_PV3_DEEP_1DG + STACK_PPSUB STR STACK_1DG + STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT_1DG + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + STATICSKY_CALIBRATION STR STATICSKY_MD_PV3_DEEPCAL +END + + + + # quick stacks @@ -1022,4 +1101,20 @@ DETREND_ONLY METADATA CHIP_PPIMAGE STR PPIMAGE_DET_ONLY + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + PSASTRO STR DEFAULT_RECIPE +END + +# Detrend Processing only for raw data +DARK_ONLY METADATA + CHIP_PPIMAGE STR PPIMAGE_OD CHIP_PSPHOT STR CHIP WARP_PSWARP STR WARP @@ -1296,12 +1391,86 @@ END +CNP_PV3 METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR CNP_ASTRO_PV3 + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL +END + +MD_PV3 METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK_1DG + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK_1DG + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR MD_ASTRO_PV3 + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_MD_DEEPCAL_PV3 +END + + # random user tests (may not be stable or consistent) TEST1 METADATA - STACK_PPSTACK STR TEST1 - STACK_PPSUB STR STACK - STACK_PSPHOT STR STACK +# CHIP_PPIMAGE STR PPIMAGE_PREMASK_PROCESS + CHIP_PPIMAGE STR APTEST_V1 + CHIP_PSPHOT STR APTEST_V1 + PSASTRO STR DEFAULT_RECIPE + FAKEPHOT STR FAKEPHOT + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + ADDSTAR STR ADDSTAR + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL + FULLFORCE_PSPHOT STR FULLFORCE_WARP +# STACK_PPSTACK STR TEST1 +# STACK_PPSUB STR STACK +# STACK_PSPHOT STR STACK END TEST2 METADATA + CHIP_PPIMAGE STR PPIMAGE_DET_ONLY + CHIP_PSPHOT STR CHIP STACK_PPSTACK STR TEST2 STACK_PPSUB STR STACK Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/system.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/system.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/system.config (revision 37823) @@ -11,4 +11,5 @@ GPC1 STR gpc1/camera.config # Pan-STARRS GPC1 GPC1-TDI STR gpc1-tdi/camera.config # Pan-STARRS GPC1 with TDI guiding + GPC2 STR gpc2/camera.config # Pan-STARRS GPC2 TC3 STR tc3/camera.config # Pan-STARRS Test Camera III ESOWFI STR esowfi/camera.config # ESO Wide-Field Imager Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/Makefile.am =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/Makefile.am (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/Makefile.am (revision 37823) @@ -3,9 +3,10 @@ install_files = \ - dvo.config \ - camera.config \ - format.config \ + dvo.config \ + camera.config \ + format.config \ format_ps2.config \ - ppImage.config + ppImage.config \ + ppMerge.config install_DATA = $(install_files) Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/camera.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/camera.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/camera.config (revision 37823) @@ -47,4 +47,12 @@ # move this elsewhere? we need a lookup table to go from filter ID to abstract name FILTER.ID METADATA + r MULTI + r STR Not available + g STR g.00001 + r STR r.00001 + i STR i.00001 + z STR z.00001 + y STR y.00001 + w STR w.00001 NONE STR NONE B STR B @@ -74,16 +82,35 @@ END -# Recipe options +# Recipe options (if not listed here, we only use the default) RECIPES METADATA - # Other recipes -# PSPHOT STR tc3/psphot.config # psphot details -# PSASTRO STR tc3/psastro.config # psastro details -# PPSTATS STR tc3/ppStats.config # ppStats recipe - PPIMAGE STR tc3/ppImage.config # ppImage recipe + PPIMAGE STR tc3/ppImage.config + PPMERGE STR tc3/ppMerge.config +# PSASTRO STR tc3/psastro.config +# PSPHOT STR tc3/psphot.config +# PSWARP STR tc3/pswarp.config +# PSVIDEOPHOT STR tc3/psvideophot.config +# REJECTIONS STR tc3/rejections.config # rejection recipe +# PPSTACK STR tc3/ppStack.config END + +# reduction classes (recipes which are grouped together) +REDUCTION STR recipes/reductionClasses.mdc FITSTYPES STR recipes/fitstypes.mdc FILERULES STR recipes/filerules-split.mdc + +EXTNAME.RULES METADATA + CMF.HEAD STR {CHIP.NAME}.hdr + CMF.DATA STR {CHIP.NAME}.psf # use .PSF and .EXT? + CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? + CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.DETEFF STR {CHIP.NAME}.deteff + + PSF.HEAD STR {CHIP.NAME}.hdr + PSF.TABLE STR {CHIP.NAME}.psf_model + PSF.RESID STR {CHIP.NAME}.psf_resid +END BLANK.HEADERS METADATA @@ -93,2 +120,5 @@ FPA.OBS STR EXPNUM END + +PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.NAME} # Rule for generating photcode + Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/format_ps2.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/format_ps2.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/format_ps2.config (revision 37823) @@ -7,5 +7,5 @@ CONTROLR STR STARGRASP INSTRUME STR tc3 - EXTEND BOOL T +# EXTEND BOOL T END @@ -304,10 +304,12 @@ # Where there might be some ambiguity, specify the format FORMATS METADATA - FPA.RA STR HOURS + FPA.RA STR DEGREES FPA.DEC STR DEGREES FPA.LONGITUDE STR HOURS FPA.LATITUDE STR DEGREES - FPA.TIME STR MJD - CELL.TIME STR MJD + FPA.TIME STR YEAR.FIRST + CELL.TIME STR YEAR.FIRST +# FPA.TIME STR MJD +# CELL.TIME STR MJD CELL.BINNING STR TOGETHER CELL.READNOISE STR ADU Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/ppImage.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/ppImage.config (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/ppImage.config (revision 37823) @@ -1,9 +1,16 @@ ### ppImage recipe configuration file for Megacam + +# binned output image options +BIN1.XBIN S32 16 +BIN1.YBIN S32 16 +BIN2.XBIN S32 150 +BIN2.YBIN S32 150 # Overscan subtraction OVERSCAN.SINGLE BOOL FALSE # Reduce overscan to a single value? +OVERSCAN.STAT STR MEDIAN +OVERSCAN.BOXCAR S32 3 OVERSCAN.FIT STR NONE # NONE | POLYNOMIAL | SPLINE OVERSCAN.ORDER S32 0 # Order of polynomial fit -OVERSCAN.STAT STR MEAN # MEAN | MEDIAN OVERSCAN BOOL TRUE # Overscan subtraction @@ -13,4 +20,7 @@ SHUTTER BOOL FALSE # Shutter correction FRINGE BOOL FALSE # Fringe subtraction + +# Normalization class +NORM.CLASS STR CHIP # How to find the per-class normalizations # this table lists extra constraints which should be applied when @@ -38,8 +48,5 @@ # only apply the fringe for these filters FRINGE.FILTERS MULTI -FRINGE.FILTERS STR r -FRINGE.FILTERS STR i -FRINGE.FILTERS STR z - +FRINGE.FILTERS STR NONE ################################################################################ @@ -50,5 +57,5 @@ CHIP METADATA OVERSCAN BOOL TRUE # Overscan subtraction - BIAS BOOL TRUE # Bias subtraction + BIAS BOOL FALSE # Bias subtraction DARK BOOL TRUE # Dark subtraction SHUTTER BOOL FALSE # Shutter correction @@ -182,5 +189,5 @@ PPIMAGE_OB METADATA OVERSCAN BOOL TRUE # Overscan subtraction - BIAS BOOL TRUE # Bias subtraction + BIAS BOOL FALSE # Bias subtraction DARK BOOL FALSE # Dark subtraction SHUTTER BOOL FALSE # Shutter correction @@ -315,2 +322,104 @@ BIN2.YBIN S32 1 # Image is already binned END + +# Overscan, bias, dark, shutter +PPIMAGE_FLATPROC_PREMASK METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK + EXPTIME STR FPA.EXPOSURE + END + END +END + +# Overscan, bias, dark, shutter, flat-field -- use FLAT_PREMASK +PPIMAGE_FLATMASK_PROCESS METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + END +END + + +# Overscan, bias, dark, shutter, flat-field -- use FLAT_PREMASK +PPIMAGE_PREMASK_PROCESS METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARK + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + DETTYPE STR FLAT_PREMASK + FILTER STR FPA.FILTERID + END + END +END Index: /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/ppMerge.config =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/ppMerge.config (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ippconfig/tc3/ppMerge.config (revision 37823) @@ -0,0 +1,72 @@ +# Mask generation --- already included in default, above +PPMERGE_DARKMASK METADATA + ITER S32 3 + MASK.SUSPECT.SIGMA F32 8.0 # Threshold for suspect pixels (sigma) + MASK.BAD F32 0.1 # Threshold for bad pixels + MASK.MODE STR FRACTION # Meaning of threshold : value, sigma + SAMPLE S32 100000 # Sampling factor for measuring the background +END + +PPMERGE_FLATMASK METADATA + ITER S32 3 + MASK.SUSPECT.SIGMA F32 3.0 # Threshold for suspect pixels (sigma) + MASK.BAD F32 0.1 # Threshold for bad pixels (sigma) + MASK.MODE STR FRACTION # Threshold for bad pixels (sigma) + SAMPLE S32 100000 # Sampling factor for measuring the background +END + +ROWS S32 32 # Number of rows to read at once + +FRINGE.NUM S32 1000 # Number of fringe regions per cell +FRINGE.SIZE S32 5 # Half-size of fringe regions +FRINGE.XSMOOTH S32 3 # Number of smoothing regions per cell in x +FRINGE.YSMOOTH S32 3 # Number of smoothing regions per cell in y +FRINGE.SMOOTH BOOL TRUE # Smooth the output image? +FRINGE.SMOOTH.SIGMA F32 2.0 # Sigma of smoothing Gaussian + +# Ordinates for fitting dark current +# example for linear in darktime only +DARK.ORDINATES METADATA + CELL.EXPOSURE S32 1 +END + +# Ordinates for fitting dark current as function of darktime and fpa temp: +# DARK.ORDINATES METADATA +# CELL.DARKTIME S32 1 +# FPA.TEMP METADATA +# ORDER S32 1 +# SCALE BOOL TRUE +# RULE STR FPA.TEMP * CELL.DARKTIME +# MIN F32 -95 +# MAX F32 -50 +# END +# END + +# Ordinates for fitting dark current as function of darktime and fpa temp: +# Counts = C0 + (CT0 + CT1*Temp)*exptime +# the 0th order term is implied, right? +# Note: GPC1 has bad values for DARKTIME in unpredictable cells; use EXPOSURE for now + +## Dark model with linear temperature dependence +## DARK.ORDINATES METADATA +## CELL.EXPOSURE S32 1 +## TEMP METADATA +## ORDER S32 1 +## RULE STR CHIP.TEMPERATURE * CELL.EXPOSURE +## END +## END + +## Dark model with quadratic temperature dependence : too sensitive to temperature range +## DARK.ORDINATES METADATA +## CELL.EXPOSURE S32 1 +## TEMP METADATA +## ORDER S32 1 +## RULE STR CHIP.TEMPERATURE * CELL.EXPOSURE +## END +## TEMP2 METADATA +## ORDER S32 1 +## RULE STR CHIP.TEMPERATURE * CHIP.TEMPERATURE * CELL.EXPOSURE +## END +## END + + Index: /branches/eam_branches/ipp-pv3-20140717-merge/mops/mops_ringsv3/populate_ringsv3_database.py =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/mops/mops_ringsv3/populate_ringsv3_database.py (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/mops/mops_ringsv3/populate_ringsv3_database.py (revision 37823) @@ -0,0 +1,82 @@ +#!/usr/bin/env python2.7 + +import MySQLdb +import pyfits +import numpy + +################################################################### +# "Parameters" +################################################################### +# Name of the table which is going to be created and populated +tableName = "skycells" +# Name of the database host +dbhost = 'localhost' +dbuser = 'mops' +dbpasswd = 'mops' +dbname = 'ringsV3' +# Name of the tesselation file +tesselation = '/local/ipp/gpc1/tess/RINGS.V3/Images.dat' +################################################################### + +def create_database(db): + cursor = db.cursor() + statement = "DROP TABLE IF EXISTS %s;" % tableName + cursor.execute(statement) + + statement = """CREATE TABLE %s ( + skycell CHAR(16) NOT NULL PRIMARY KEY COMMENT 'Skycell ID (aka Name in Images.dat file)', + raProjectionCenter DOUBLE NOT NULL COMMENT 'Right Ascension of the center of projection cell (in degrees, aka crval1)', + craProjectionCenter DOUBLE NOT NULL COMMENT 'Cosine of ra of center of projection cell', + sraProjectionCenter DOUBLE NOT NULL COMMENT 'Sine of ra of center of projection cell', + decProjectionCenter DOUBLE NOT NULL COMMENT 'Declination of center of projection cell (in degrees, aka crval2)', + cdecProjectionCenter DOUBLE NOT NULL COMMENT 'Cosine of decl of center of projection cell', + sdecProjectionCenter DOUBLE NOT NULL COMMENT 'Sine of decl of center of projection cell', + xSkycellCenter DOUBLE NOT NULL COMMENT 'X coordinate of the skycell center (in pixels, aka crpix1)', + ySkycellCenter DOUBLE NOT NULL COMMENT 'Y coordinate of the skycell center (in pixels, aka crpix2)' +) ENGINE=InnoDB COMMENT='Contents of tesselation used for MOPS diffs mapped to a database table. Notes: (a) Scales on both X and Y axis (aka cdelt1, cdelt2) are 5 arcseconds. (b) Rotation coordinates (aka pc[12]_[12]) in from (ra,dec) coordinates to image plane coordinates are ((-1,0),(0,1)). (c) The database has been populated on 2014-10-20 by schastel using /local/ipp/gpc1/tess/RINGS.V3/Images.dat whose md5sum is 8eb517fe2e894e66488d154f5bb7c030; (d) Associated software: SVN repository: https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/mops/mops_ringsv3' ; +""" % (tableName) + cursor.execute(statement) + cursor.close() + +def populate_database(database): + fits = pyfits.open(tesselation) + table = fits[1].data + raProjectionCenterRadians = numpy.radians(table.field('CRVAL1')) + craProjectionCenter = numpy.cos(raProjectionCenterRadians) + sraProjectionCenter = numpy.sin(raProjectionCenterRadians) + decProjectionCenterRadians = numpy.radians(table.field('CRVAL2')) + cdecProjectionCenter = numpy.cos(decProjectionCenterRadians) + sdecProjectionCenter = numpy.sin(decProjectionCenterRadians) + data = [ table.field('NAME'), + table.field('CRVAL1'), craProjectionCenter, sraProjectionCenter, + table.field('CRVAL2'), cdecProjectionCenter, sdecProjectionCenter, + table.field('CRPIX1'), table.field('CRPIX2'), + ] + query = """INSERT INTO %s (skycell, +raProjectionCenter, craProjectionCenter, sraProjectionCenter, +decProjectionCenter, cdecProjectionCenter, sdecProjectionCenter, +xSkycellCenter, ySkycellCenter) VALUES (%%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s)""" % tableName + # print query + remapped = zip(*data) + # print + # print query % remapped[0] + # print + # print remapped[0:5] + begin = 0 + step = 10000 + while begin < len(remapped): + cursor = database.cursor() + cursor.executemany(query, remapped[begin:begin+step]) + cursor.close() + database.commit() + begin += step + cursor.executemany(query, remapped[begin:-1]) + fits.close() + cursor.close() + +if __name__ == "__main__": + database = MySQLdb.connect(dbhost, dbuser, dbpasswd, dbname) + create_database(database) + populate_database(database) + database.commit() + database.close() Index: /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMops.h =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMops.h (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMops.h (revision 37823) @@ -6,5 +6,5 @@ #define IN_EXTNAME "SkyChip.psf" // Extension name for data in input -#define OBSERVATORY_CODE "F51" // IAU Observatory Code +// #define OBSERVATORY_CODE "F51" // IAU Observatory Code #define OUT_EXTNAME "MOPS_TRANSIENT_DETECTIONS" // Extension name for data in output #define SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_BADPSF | PM_SOURCE_MODE_SATURATED | \ @@ -23,4 +23,6 @@ psS64 warp_id; // Warp stage identifier psS64 diff_id; // Diff stage identifier + psString camera; // Camera name + psString obscode; // observatory code (derived from camera) bool positive; // Sense of subtraction, T=positive, F=negative float zp, zpErr; // Magnitude zero point and error @@ -59,5 +61,6 @@ psVector *raExtErr, *decExtErr; // Error in sky coordinates for RA_EXT and DEC_EXT psVector *mask; // Mask for detections - float platescale; //Plate scale at centroid + float platescale; // Plate scale at centroid + psString fpashutoutc; // FPA shutoutc } ppMopsDetections; Index: /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsArguments.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsArguments.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsArguments.c (revision 37823) @@ -42,4 +42,6 @@ static void mopsArgumentsFree(ppMopsArguments *args) { + psFree(args->camera); + psFree(args->obscode); psFree(args->input); psFree(args->exp_name); @@ -63,4 +65,6 @@ args->warp_id = 0; args->diff_id = 0; + args->camera = NULL; + args->obscode = NULL; args->zp = NAN; args->positive = true; @@ -94,4 +98,5 @@ psMetadataAddS64(arguments, PS_LIST_TAIL, "-warp_id", 0, "Warp stage identifier", 0); psMetadataAddS64(arguments, PS_LIST_TAIL, "-diff_id", 0, "Diff stage identifier", 0); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-camera", 0, "Camera name", "No comment provided"); psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Inverse subtraction?", false); psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp", 0, "Magnitude zero point", NAN); @@ -126,4 +131,5 @@ args->warp_id = psMetadataLookupS64(NULL, arguments, "-warp_id"); args->diff_id = psMetadataLookupS64(NULL, arguments, "-diff_id"); + args->camera = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-camera")); args->positive = !psMetadataLookupBool(NULL, arguments, "-inverse"); // NOTE: negated args->zp = psMetadataLookupF32(NULL, arguments, "-zp"); @@ -137,4 +143,19 @@ args->shutoutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-shutoutc")); + if (!args->camera) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "camera name not provided."); + return NULL; + } + if (!strcasecmp(args->camera, "GPC1")) { + args->obscode = psStringCopy("F51"); + } + if (!strcasecmp(args->camera, "GPC2")) { + args->obscode = psStringCopy("F52"); + } + if (!args->obscode) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "unknown camera name."); + return NULL; + } + psTrace("ppMops.args", 1, "Done parsing command-line arguments\n"); Index: /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsDetections.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsDetections.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsDetections.c (revision 37823) @@ -20,4 +20,5 @@ psFree(det->decErr); psFree(det->mask); + psFree(det->fpashutoutc); return; } @@ -48,4 +49,5 @@ det->mask = NULL; det->diffSkyfileId = 0; + det->fpashutoutc = NULL; return det; } Index: /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsRead.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsRead.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsRead.c (revision 37823) @@ -105,4 +105,5 @@ det->naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); // Number of columns det->naxis2 = psMetadataLookupS32(NULL, header, "IMNAXIS2"); // Number of rows + det->fpashutoutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.SHUTOUTC")); psFree(header); @@ -241,7 +242,8 @@ long numGood = 0; // Number of good rows for (long row = 0; row < size; row++) { - psU32 flags = flagsV->data.U32[row]; // psFitsTableGetU32(NULL, table, row, "FLAGS"); + //psU32 flags = flagsV->data.U32[row]; // psFitsTableGetU32(NULL, table, row, "FLAGS"); + psU64 flags = flagsV->data.U64[row]; // table reads in as U64 if (flags & SOURCE_MASK) { - psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags: %ud", row, i, flags); + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags: %lud", row, i, flags); det->mask->data.U8[row] = 0xFF; continue; Index: /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsWrite.c =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsWrite.c (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/ppTranslate/src/ppMopsWrite.c (revision 37823) @@ -89,5 +89,5 @@ psWarning("no inputs with surviving detections. output header will be incomplete"); } - psMetadataAddStr(header, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE); + psMetadataAddStr(header, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", args->obscode); psMetadataAddF32(header, PS_LIST_TAIL, "MAGZP", 0, "Magnitude zero point", args->zp); //MOPS want the name ZEROPOINT for MAGZP @@ -102,4 +102,5 @@ psMetadataAddF32(header, PS_LIST_TAIL, "PLTSCALE_EXT", 0, "Plate scale at centroid", det->platescale); psMetadataAddF32(header, PS_LIST_TAIL, "PLTSCALE", 0, "Plate scale at centroid", det->platescale); + psMetadataAddStr(header, PS_LIST_TAIL, "FPA.SHUTOUTC", 0, "Time of exposure", det->fpashutoutc); //field in header that tells about the CMF version Index: /branches/eam_branches/ipp-pv3-20140717-merge/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/pstamp/scripts/pstamp_checkdependent.pl (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/pstamp/scripts/pstamp_checkdependent.pl (revision 37823) @@ -24,4 +24,7 @@ my $ra_max_3PI_hours = 22.5; # disable updates for skycells with ra greater than this (hours) my $ra_max_3PI = $ra_max_3PI_hours * 15 * 3.14149 / 180.; # rawExp.ra is in radians + +# verbose flag for parse commands +my $parse_verbose = 0; # XXX: put this in a module somewhere @@ -49,4 +52,5 @@ 'dbserver=s' => \$ps_dbserver, # postage stamp server dbserver 'verbose' => \$verbose, + 'parse-verbose' => \$parse_verbose, 'save-temps' => \$save_temps, 'no-update' => \$no_update, @@ -78,4 +82,6 @@ } +print "\n==== Starting dependency checking for $dep_id $stage $stage_id $component at " . (scalar localtime) . ". ====\n"; + if ($label) { # rlabel is deprecated. Use one based on the supplied label parameter which is the current label @@ -84,4 +90,5 @@ # (and formerly in pstampparse.pl) is not particularly clean but it's simple. my $new_rlabel = 'ps_ud_' . $label; + $rlabel = "undefined" if !defined $rlabel; if ($new_rlabel ne $rlabel) { print "Notice: using $new_rlabel instead of $rlabel for update label.\n"; @@ -120,5 +127,5 @@ } -my $it = runToolAndParseExpectOne($cmd, $verbose); +my $it = runToolAndParseExpectOne($cmd, $parse_verbose); my_die("no components found", $PS_EXIT_PROG_ERROR) if ( !$it); @@ -136,4 +143,5 @@ } + # magic is no longer rquired $need_magic = 0; @@ -142,4 +150,5 @@ if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full')) { + print "\nDependency Satisfied for $stage $stage_id $component\n"; # This Dependency is satisfied. All done. Release the pstampJobs # @@ -178,5 +187,5 @@ } - # assume the fault is transient. + # hope the fault is transient. # fault the dependent to give the fault a chance to correct itself my_die("Component faulted on update dep_id: $dep_id", $PS_EXIT_SYS_ERROR); @@ -279,5 +288,5 @@ ($fault eq $PSTAMP_GONE)) { - print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state fault: $fault cannot update\n"; + print "chipRun state is $chip->{chip_id} has state: $state class_id $chip->{class_id} data_state: $data_state fault: $fault cannot update.\n"; my $error_code; if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) { @@ -291,10 +300,10 @@ } elsif ($chip->{state} eq 'goto_cleaned') { - # we don't need to check for full data_state - # if ($chip->{data_state} eq 'full') { { + print "dependent chip is in state goto_cleaned, changing state to update.\n"; + # cleanup must not be running. Set state to update. If this chip is not 'full' it will be # set to be updated the next time this script is invoked - # XXX: In the very rare case that this chip run is actually in the running cleanup pantasks + # XXX: In the very unlikely case that this chip run is actually in the running cleanup pantasks # queue things may get confused but we can live with that my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id"; @@ -313,4 +322,6 @@ } } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) { + + print "Setting chip imfile $chip_id $chip->{class_id} to be updated.\n"; # chiptool does more state checking to insure this isn't done prematurely. @@ -343,4 +354,5 @@ # we've had a number of runs in this limbo state + print "Changing chip run $chip_id to state update for $chip->{class_id} which is in data_state update.\n"; my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id"; $command .= " -set_label $rlabel" if $rlabel; @@ -364,5 +376,5 @@ return $PSTAMP_GONE; } elsif ($fault_count > $max_fault_count) { - print "$stage $stage_id has faulted $fault_count times. Giving up\n"; + print "Dependency for $stage $stage_id has been found in fault state $fault_count times. Giving up. Fault: $fault\n"; # XXX: stop faulting chips for now # $fault = $PSTAMP_GONE; @@ -421,5 +433,5 @@ if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) { - print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n"; + print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs.\n"; my $error_code; if (($state eq 'error_cleaned') or ($data_state eq 'error_cleaned')) { @@ -430,12 +442,14 @@ return $error_code } - if (($data_state eq 'full') and ($metadata->{fault})) { + if ((($data_state eq 'full') or ($data_state eq 'update')) and ($metadata->{fault})) { # fault dependent. my $fault = $metadata->{fault}; - print STDERR "warp $warp_id $skycell_id faulted: $fault"; + print STDERR "warp $warp_id $skycell_id faulted with code: $fault."; + # XXX: TODO maybe: If ($fault != 2 and $fault != $PSTAMP_GONE) revert the skycell. The warp task only reverts fault 2 + # and sometimes warps will fault due to nfs problems that end up with fault = 4;. return $fault; } if ($metadata->{quality} ne 0) { - print STDERR "warp $warp_id $skycell_id bad quality on update: $metadata->{quality}\n"; + print STDERR "warp $warp_id $skycell_id bad quality on update: $metadata->{quality}.\n"; return $PSTAMP_GONE; } @@ -448,4 +462,5 @@ # It's "safe" to set the state to update. If the skycell is not full it will be set to update # the next time this task runs + print "Change state of warp $warp_id to update.\n"; my $command = "$warptool -updaterun -set_state update -warp_id $warp_id"; $command .= " -set_label $rlabel" if $rlabel; @@ -465,10 +480,10 @@ # get the list of input chips for this skycell my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; - my $data = runToolAndParse($command, $verbose); + my $data = runToolAndParse($command, $parse_verbose); if (!$data or scalar @$data == 0) { # This happens if the chipProcessedImfile disappears which happened when earlier # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before # deleting the row. - print STDERR "failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id"; + print STDERR "failed to find warpSkyCellMap for warpRun $warp_id skycell_id $skycell_id"; return $PSTAMP_GONE; } @@ -478,4 +493,5 @@ my $chip_id; foreach my $chip (@$data) { + # XXX: change tools to include cam_state (camRun.state). If it is not full the warp updates will never run. my $cam_fault = $chip->{cam_fault}; if (defined $cam_fault and $cam_fault > 0) { @@ -488,7 +504,7 @@ $chip->{decl} = $metadata->{decl}; +if (0) { # if chip has been magicked before require it to be magicked again # because the warp pending query requires it. -if (0) { if ($chip->{magicked} < 0) { print "Input has been destreaked so we must destreak before warping\n"; @@ -514,4 +530,5 @@ # the reason we defer setting the warp to update is so that we can handle error conditions at previous # stages more easily. + print "Chips are ready for warp $warp_id $skycell->{skycell_id}. Setting skycell to be updated.\n"; my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; $command .= " -set_label $rlabel" if $rlabel; @@ -527,4 +544,5 @@ } } elsif ($chips_ready and $skycell->{data_state} eq 'update' and $skycell->{state} ne 'update') { + print "Chips are ready for warp $warp_id $skycell->{skycell_id} but run state is $skycell->{state}. Changing state to update.\n"; my $command = "$warptool -updaterun -warp_id $warp_id -set_state update"; $command .= " -set_label $rlabel" if $rlabel; @@ -566,9 +584,8 @@ my $skycell_id = $skycell->{skycell_id}; - # if ($metadata->{state} eq 'goto_cleaned' and $metadata->{data_state} eq 'full') { if ($metadata->{state} eq 'goto_cleaned') { # cleanup must not be running. Set state to update. If the skycell is not 'full' it will be - # set to be updated the next time this script is invoked - print "changing state of diffRun $diff_id from goto_cleaned to update. skycell $skycell_id is full\n"; + # set to be updated the next time this script is invoked. + print "Changing state of diffRun $diff_id from goto_cleaned to update.\n"; my $command = "$difftool -updaterun -set_state update -diff_id $diff_id"; $command .= " -set_label $rlabel" if $rlabel; @@ -589,5 +606,5 @@ # check the state of the template stack my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; - my $stack = runToolAndParseExpectOne($command, $verbose); + my $stack = runToolAndParseExpectOne($command, $parse_verbose); my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; @@ -597,8 +614,9 @@ return $PSTAMP_GONE; } + print " Stack $skycell->{stack2} is ready.\n"; # now check the warp $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; - my $warp = runToolAndParseExpectOne($command, $verbose); + my $warp = runToolAndParseExpectOne($command, $parse_verbose); my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; @@ -615,7 +633,8 @@ } # warps are ready fall through and queue the diff update + print " Warp $skycell->{warp1} $skycell_id is ready.\n"; } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) { my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; - my $warp1 = runToolAndParseExpectOne($command, $verbose); + my $warp1 = runToolAndParseExpectOne($command, $parse_verbose); my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1; @@ -637,5 +656,5 @@ } $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; - my $warp2 = runToolAndParseExpectOne($command, $verbose); + my $warp2 = runToolAndParseExpectOne($command, $parse_verbose); my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2; @@ -661,5 +680,5 @@ # check the state of the input stack my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}"; - my $stack1 = runToolAndParseExpectOne($command, $verbose); + my $stack1 = runToolAndParseExpectOne($command, $parse_verbose); my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1; @@ -671,5 +690,5 @@ # check the state of the template stack $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; - my $stack2 = runToolAndParseExpectOne($command, $verbose); + my $stack2 = runToolAndParseExpectOne($command, $parse_verbose); my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2; @@ -684,5 +703,5 @@ # check the state of the input stack my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}"; - my $stack = runToolAndParseExpectOne($command, $verbose); + my $stack = runToolAndParseExpectOne($command, $parse_verbose); my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; @@ -695,5 +714,5 @@ # now check the template warp $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; - my $warp = runToolAndParseExpectOne($command, $verbose); + my $warp = runToolAndParseExpectOne($command, $parse_verbose); my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; @@ -736,4 +755,6 @@ print "skipping $command\n"; } + } else { + print " diff is in update state\n"; } @@ -763,5 +784,7 @@ my $now = DateTime->now->mjd; my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.; - print "Time to run $program: $dtime_tool\n"; + if ($dtime_tool > 0.1) { + print "Time to run $program: $dtime_tool\n"; + } my $buf = join "", @$stdout_buf; @@ -777,5 +800,7 @@ my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.; - print "Time to parse results from $program: $dtime_parse\n"; + if ($dtime_parse > 0.1) { + print "Time to parse results from $program: $dtime_parse\n"; + } return $results; @@ -848,5 +873,5 @@ if (!$dsRun_state) { my $command = "$chiptool -listrun -chip_id $stage_id"; - my $data = runToolAndParse($command, $verbose); + my $data = runToolAndParse($command, $parse_verbose); my $chipRun = $data->[0]; $dsRun_state = $chipRun->{dsRun_state}; @@ -862,5 +887,5 @@ foreach my $c (@$components) { my $command = "$magicdstool -destreakedfile -magic_ds_id $magic_ds_id -component $c"; - my $dsfile = runToolAndParseExpectOne($command, $verbose); + my $dsfile = runToolAndParseExpectOne($command, $parse_verbose); if (!$dsfile) { my_die("failed to find magicDSFile for ${stage}Run $stage_id $c", $PS_EXIT_UNKNOWN_ERROR); Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci0.pl =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci0.pl (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci0.pl (revision 37823) @@ -0,0 +1,6 @@ +@num = ('00','01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19'); + +foreach $numb (@num) { + $thing = `ssh stsci$numb 'df -h | grep stsci$numb.0'`; + print $thing; +} Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci1.pl =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci1.pl (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci1.pl (revision 37823) @@ -0,0 +1,6 @@ +@num = ('00','01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19'); + +foreach $numb (@num) { + $thing = `ssh stsci$numb 'df -h | grep stsci$numb.1'`; + print $thing; +} Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci2.pl =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci2.pl (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/check_stsci2.pl (revision 37823) @@ -0,0 +1,6 @@ +@num = ('00','01','02','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19'); + +foreach $numb (@num) { + $thing = `ssh stsci$numb 'df -h | grep stsci$numb.2'`; + print $thing; +} Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/queuesliceslap2.pl =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/queuesliceslap2.pl (revision 37822) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/queuesliceslap2.pl (revision 37823) @@ -4,19 +4,28 @@ -$label = "LAP.ThreePi.20130717"; - +$label1 = "LAP.ThreePi.20130717"; +$label2 = "LAP.ThreePi.20130717.pole3"; +$label3 = "LAP.ThreePi.20130717.pole3.CNP"; +#heather done bad things to segments and slices... +@labels = ($label1, $label2, $label3); @segments = (0,1,2,3,4,5,6,7,8); +#@segments=(0,1); @segments1 = ( -35, -22,-10,2,14,26,38,50,62); @segments2 = ( -22,-10,2,14,26,38,50,62,75); -@slices = ("00","02","04","06","08",10,12,14,16,18,20,22); - -$slice = $slices[0]; -$slice = $slices[1]; +@slices = ("00","02","04","06","08","10","12","14","16","18","20","22"); +#@slices = ("00","02","04"); + +foreach $label (@labels) { + +foreach $slice (@slices) { + +#$slice = $slices[0]; +#$slice = $slices[1]; foreach $num (@segments) { - $dvodb = "LAP.20130717"; - $minidvodb_group = "LAP.slice$slice.$segments[$num].cam"; + $dvodb = "LAP.PV2.V2.20130717"; + $minidvodb_group = "LAP.PV2.V2.slice$slice.$segments[$num].cam"; $dvodb_label = "$minidvodb_group"; $ra_min = $slice*15.0; @@ -27,7 +36,12 @@ - print "addtool -definebyquery -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $dvodb_label -stage cam -uncensored -ra_min $ra_min -ra_max $ra_max -decl_min $dec_min -decl_max $dec_max -dbname gpc1 \n"; - - + $com = "addtool -definebyquery -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $dvodb_label -stage cam -uncensored -ra_min $ra_min -ra_max $ra_max -decl_min $dec_min -decl_max $dec_max -dbname gpc1"; + print "$com\n"; + `$com`; + } } +$com2 = "addtool -definebyquery -label $label -set_dvodb $dvodb -set_minidvodb_group LAP.PV2.V2.pole.cam -set_minidvodb -set_label LAP.PV2.V2.pole.cam -stage cam -uncensored -ra_min 0 -ra_max 360 -decl_min 75 -decl_max 90 -dbname gpc1"; +print "$com2\n"; +`$com2`; +} Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/queuesliceslap2skycal.pl =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/queuesliceslap2skycal.pl (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/heather/queuesliceslap2skycal.pl (revision 37823) @@ -0,0 +1,43 @@ +#addtool -pretend -definebyquery -label LAP.ThreePi.20120706 -set_dvodb LAP.20120706 -set_minidvodb_group LAP.slice06.part2.cam -set_minidvodb -set_label LAP.slice06.part2.cam -stage cam -uncensored -ra_min 90 -ra_max 120 -decl_min -35 -decl_max 20 -dbname gpc1 -simple | wc + +#addtool -pretend -definebyquery -label LAP.ThreePi.20120706 -set_dvodb LAP.20120706 -set_minidvodb_group LAP.slice06.cam -set_minidvodb -set_label LAP.slice06.cam -stage cam -uncensored -ra_min 90 -ra_max 120 -decl_min 20 -decl_max 75 -dbname gpc1 -simple | wc + + +$label1 = "LAP.ThreePi.20130717"; +$label2 = "LAP.ThreePi.20130717.pole3"; +$label3 = "LAP.ThreePi.20130717.pole3.CNP"; +@labels = ($label1,$label2,$label3); +@segments = (0,1,2,3,4,5,6,7,8); +@segments1 = ( -35, -22,-10,2,14,26,38,50,62); +@segments2 = ( -22,-10,2,14,26,38,50,62,75); + + +@slices = ("00","02","04","06","08","10","12","14","16","18","20","22"); +foreach $label (@labels) { +foreach $slice (@slices) { + +#$slice = $slices[0]; +#$slice = $slices[1]; + foreach $num (@segments) { + + $dvodb = "LAP.PV2.V2.20130717"; + $minidvodb_group = "LAP.PV2.V2.slice$slice.$segments[$num].skycal"; + $dvodb_label = "$minidvodb_group"; + $ra_min = $slice*15.0; + $ra_max = ($slice+2)*15.0; + $dec_min = $segments1[$num]; + $dec_max = $segments2[$num]; +# print "$dvodb $minidvodb_group $ra_min $ra_max $dec_min $dec_max\n"; + + + $com = "addtool -definebyquery -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $dvodb_label -stage skycal -uncensored -ra_min $ra_min -ra_max $ra_max -dec_min $dec_min -dec_max $dec_max -dbname gpc1"; + print "$com\n"; + `$com`; + } + + +} +$com2 = "addtool -definebyquery -label $label -set_dvodb $dvodb -set_minidvodb_group LAP.PV2.V2.pole.skycal -set_minidvodb -set_label LAP.PV2.V2.pole.skycal -stage skycal -uncensored -ra_min 0 -ra_max 360 -dec_min 75 -dec_max 90 -dbname gpc1"; +print "$com2\n"; +`$com2`; +} Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/README =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/README (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/README (revision 37823) @@ -0,0 +1,70 @@ +# IASC is the asteroids/comets outreach program +# Ask Richard Wainscoat about it +# SVN: https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/tools/iasc + +### Usage +1) Select the set of exposures to be published + e.g. o6897g0312o o6897g0328o o6897g0344o o6897g0360o +2) Choose a name for the target directory (usually the date when the observations + were made) + e.g. 20140828 +3) Execute: + ./RUN.sh + e.g. + ./RUN.sh 20140828 o6897g0312o o6897g0328o o6897g0344o o6897g0360o +4) Send an e-mail to the people who requested the set of exposures, telling them + where to find the set + + +### Details + +To get chip processed image files with updated astrometry we have the postage stamp +server make full image stamps for each chip. Then we download the files and change +the names to remove the stuff that the pstamp server needs to add. + +The process is + +1. Make the destination directory. I have been using the date. For example + mkdir 20120412 + +For each exposure + postreq + # wait for request to finish (check pstamp status page) + getit + +2. rsync the directory to /data/ipp022.0/ps1-outreach: + rsync -avz YYYYMMDD ipp@ipp0222:/data/ipp022.0/ps1-outreach/ + To speed it up: + rsync -av YYYYMMDD ipp@ipp0222:/data/ipp022.0/ps1-outreach --exclude 'YYYYMMDD/*/*.tmp' + +3. The directory is visible to epo folks as + rsync://ipp0222.ifa.hawaii.edu/ps1-outreach/YYYYMMDD + Test it with: + rsync -avz rsync://ps1epo@ipp0222.ifa.hawaii.edu/ps1-outreach/20130308 . + Password can be found in ipp022:/etc/rsyncd.secrets + +4. Make sure that the smfs have been published to the datastore + ./check_if_smfs_have_been_published.py o6359g0309o o6359g0330o o6359g0351o o6359g0372o + +5. Tell the persons who made the request that all is good + +###### +For example + (ippc18:~/outreach) ./postreq o6029g0262o + running psmkreq --req_type byexp --stage chip --pixcenter --x 0 --y 0 --width 0 --height 0 --id o6029g0262o --req_name o6029g0262o.20120412T201043 + running pstamptool -dbname ippRequestServer -dbserver ippc17 -label WEB -addreq -uri /home/panstarrs/bills/outreach/o6029g0262o.20120412T201043.fits + posted o6029g0262o.20120412T201043 req_id: 160657 + results will be at http://datastore.ipp.ifa.hawaii.edu/pstampresults/o6029g0262o.20120412T201043 + + ./getit o6029g0262o.20120412T201043 20120412 + # lots of noise from dsget + + # files are in 20120412/o6029g0262o + +./check_if_smfs_have_been_published.py o6359g0309o o6359g0330o o6359g0351o o6359g0372o +o6359g0309o: OK (2 published) +o6359g0330o: OK (2 published) +o6359g0351o: OK (2 published) +o6359g0372o: OK (2 published) + +NOTE : the directory in which this is run needs to be globally visible via NFS. Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/RUN.sh =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/RUN.sh (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/RUN.sh (revision 37823) @@ -0,0 +1,46 @@ +#!/bin/bash + +targetdir=$1 +shift +exposures=$* + +echo "Creating targetdir [$targetdir]" +mkdir -p $targetdir +mkdir -p $targetdir/smf + +images="" +for exposure in $exposures; do + echo "Requesting images for $exposure" + image=`./postreq $exposure | grep datastore.ipp.ifa.hawaii.edu | sed 's/^.*\///'` + images="$images $image" + echo "Requesting smf for $exposure" + ./get_smf.sh $exposure $targetdir +done + +todelete="" +for image in $images; do + status=1 + while [ $status -ne 0 ]; do + sleep 30 + echo "Copying images for $image" + ./getit $image $targetdir + status=$? + done + todelete="$todelete $image.fits" +done +echo "All images copied locally" + +echo "Rsyncing: rsync -av $targetdir ipp@ipp0222:/data/ipp022.0/ps1-outreach/" +rsync -av $targetdir ipp@ipp0222:/data/ipp022.0/ps1-outreach/ +rsync -av $targetdir ipp@ipp0222:/data/ipp022.0/ps1-outreach/ + +echo +echo "Done" +echo "Dataset can be copied from rsync://ps1epo@ipp0222.ifa.hawaii.edu/ps1-outreach/$targetdir" +echo "Command line:" +echo " rsync -avz rsync://ps1epo@ipp0222.ifa.hawaii.edu/ps1-outreach/$targetdir ." +echo +echo "Command to run to cleanup local directory:" +echo " rm -rf $targetdir $todelete" +echo + Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/get_smf.sh =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/get_smf.sh (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/get_smf.sh (revision 37823) @@ -0,0 +1,12 @@ +#!/bin/bash + +# $1: expsore name +# $2: destination directory (date) + +pathbase=`echo "SELECT path_base FROM camProcessedExp JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE exp_name = '$1' " | mysql -h scidbm -u ipp -pipp gpc1 | grep -v path_base` + +source=`neb-ls -p $pathbase.smf` + +mkdir -p $2/smf +cp $source $2/smf/$1.smf + Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/getit =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/getit (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/getit (revision 37823) @@ -0,0 +1,49 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +my $dsroot = 'http://datastore.ipp.ifa.hawaii.edu/pstampresults'; + +my $fs = shift; +my $dest = shift; + +# set this if the files have already been downloaded +my $skip_download = 0; + +die "usage: $0 > 8; + die "dsgetfileset failed with $rc $status\n"; + } +} + +# read exp_name from results.mdc (keyword is FPA_ID) +my $grepout = `grep FPA_ID $destdir/results.mdc | tail -n 1`; +chomp $grepout; + +die "failed to find exp_name in results.mdc" unless $grepout; + +my (undef, undef, $exp_name) = split " ", $grepout; + +die "failed to find exp_name in results.mdc: $grepout" unless $exp_name; + +my $final_destdir = "$dest/$exp_name"; +rename $destdir, $final_destdir or die "failed to rename $destdir"; + +chdir $final_destdir or die "failed to chdir to $final_destdir"; + +system 'perl /data/ippc18.0/home/ipp/iasc/rename.stamp.outputs.pl *'; + + Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/obsolete/check_if_smfs_have_been_published.py =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/obsolete/check_if_smfs_have_been_published.py (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/obsolete/check_if_smfs_have_been_published.py (revision 37823) @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import MySQLdb +import sys + +def usage(program_name): + return """ +%s [exposure name] [...] + +Check if the smf of have been published to the datastore + +-h, -help, --help: + Displays this help and exits +""" + +def process_arguments(arguments): + if len(arguments)<2: + sys.stderr.write(usage(arguments[0])) + sys.exit(1) + index = 1 + if arguments[index] == "-h" or arguments[index] == "-help" or arguments[index] == "--help": + sys.stderr.write(usage(arguments[0])) + sys.exit(0) + return arguments[index:] + +if __name__ == "__main__": + exposures = process_arguments(sys.argv) + gpc1 = MySQLdb.connect('ippdb01', 'ipp', 'ipp', 'gpc1') + cursor = gpc1.cursor() + for exposure in exposures: + statement = "SELECT COUNT(distinct(distRun.dist_id)) FROM distRun JOIN distComponent USING(dist_id) JOIN camRun ON camRun.cam_id=distRun.stage_id JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE exp_name = '%s' AND distRun.stage='camera'" % exposure + cursor.execute(statement) + count=cursor.fetchall()[0][0] + if count != 0: + print "%s: OK (%d published)" % (exposure, count) + else: + print "%s: FAIL (%d published)" % (exposure, count) + Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/postreq =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/postreq (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/postreq (revision 37823) @@ -0,0 +1,36 @@ +#!/usr/bin/env perl + +# create a postage stamp request file to get full chip images for +# a given exposure +# post it to the server with label WEB + +use strict; +use warnings; +use POSIX qw( strftime ); + +my $exp_name = shift; +die "need exposure name\n" if !$exp_name; + +my $datestr = strftime "%Y%m%dT%H%M%S", gmtime; + +# request name must be unique add datetime +my $req_name = "$exp_name.$datestr"; + +# make the request file +my $cmd = "psmkreq --req_type byexp --stage chip --pixcenter --x 0 --y 0 --width 0 --height 0 --id $exp_name --req_name $req_name"; + +print "running $cmd\n"; +my $rc = system $cmd; + +die "psmkreq failed" if $rc; + +# add the request to the database +my $postcmd = "pstamptool -dbname ippRequestServer -dbserver ippc17 -label WEB -addreq -uri $ENV{PWD}/$req_name.fits"; + +print "running $postcmd\n"; + +my $req_id = `$postcmd`; +chomp $req_id; + +print "posted $req_name req_id: $req_id\n"; +print "results will be at http://datastore.ipp.ifa.hawaii.edu/pstampresults/$req_name\n"; Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/rename.stamp.outputs.pl =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/rename.stamp.outputs.pl (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/iasc/rename.stamp.outputs.pl (revision 37823) @@ -0,0 +1,12 @@ +#!/usr/bin/env perl + +my $verbose = 0; + +foreach my $name (@ARGV) { + next if !($name =~ /1_/); + my (undef, undef, undef, $new_name) = split "_", $name; + die "failed to extract new name from $name\n" unless $new_name; + print "$name $new_name\n" if $verbose; + rename $name, $new_name or die "failed to rename $name\n"; +} + Index: /branches/eam_branches/ipp-pv3-20140717-merge/tools/stask_chip.py =================================================================== --- /branches/eam_branches/ipp-pv3-20140717-merge/tools/stask_chip.py (revision 37823) +++ /branches/eam_branches/ipp-pv3-20140717-merge/tools/stask_chip.py (revision 37823) @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +import os +import time +import sys +import subprocess + +def emit_output(p): + (sout, serr) = p.communicate(None) + if len(sout): + print >> sys.stdout, sout, + sys.stdout.flush() + +def wait(n): + global n_completed + while (len(procs) >= n): + t = time.time() + for p in procs: + if p.poll() is not None: + print >> sys.stderr, '%.2f -- %s is done, status %d' % (t-times[p], cmds[p].rstrip(), p.returncode) + emit_output(p) + n_completed=n_completed+1 + busy[prochost[p]] = 0 + del prochost[p] + del cmds[p] + del times[p] + procs.remove(p) + elif t-times[p] > timeout: + print >> sys.stderr, '%s timed out!' % cmds[p].rstrip() + p.terminate() + emit_output(p) + busy[prochost[p]] = 0 + del prochost[p] + del cmds[p] + del times[p] + procs.remove(p) + time.sleep(1.0) + time.sleep(0.05) + +def free_host(): + for host in hosts: + if busy[host] != 1: + return host + return None + +task_file = sys.argv[1] +hosts_file = sys.argv[2] +nprocs = int(sys.argv[3]) # Oversubscription parameter: number of processors to devote to a task + +hosts = [] +tasks = [] +nname = -1 # prefix of host specification +procs = [] # process, defined as task on a host +cmds = {} # command assocated with process +prochost = {} # host associate with process +times = {} +busy = {} +timeout = 2*3600 +n_active = 0 # number of active jobs we expect to have running should equal Nhosts * Ncore_per_host / nprocs + + +n_completed=0 +start_time = time.time() + +with open(task_file) as f: + for task in f: + if task[0] != '#': + tasks.append(task.rstrip()) + +with open(hosts_file) as f: + for host in f: + if host[:2] == 'mu': + cores = int(12 / nprocs) # Use nprocs to reduce the core subscription + nname = 6 + elif host[:2] == 'ml': + cores = 16 + nname = 5 + elif host[:2] == 'mp': + cores = 8 + nname = 5 + else: + cores = 1 + nname = -1 + for core in range(cores): + h = "%s-%d" % (host[:nname], core) + hosts.append(h) + busy[h] = 0 + n_active = n_active + 1 + +print 'running %d commands at a time from %d tasks in %s' \ + % (n_active, len(tasks), task_file) + +for task in tasks: + h = free_host() + busy[h] = 1 + command = '/usr/bin/ssh -n -q %s \"cd /scratch3/watersc1/; %s\"' % (h[:nname], task) + print 'executing ', command + sys.stdout.flush() + try: + p = subprocess.Popen(command, bufsize=-1, shell=True, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except: + print >> sys.stderr, "Exception! %s" % command + procs.append(p) + prochost[p] = h + cmds[p] = command + times[p] = time.time() + wait(n_active) + +wait(1) + +t = time.time() +print >> sys.stderr, 'all done. %d tasks completed in %2.f seconds (%.2f per second)' % (n_completed, t-start_time, n_completed/(t-start_time))