Index: /branches/eam_branches/ipp-20230313/ippScripts/scripts/xccal.pl =================================================================== --- /branches/eam_branches/ipp-20230313/ippScripts/scripts/xccal.pl (revision 42662) +++ /branches/eam_branches/ipp-20230313/ippScripts/scripts/xccal.pl (revision 42663) @@ -51,6 +51,4 @@ print " --outroot (root) : Output root name\n"; print " --path_base (path_base) : path_base of input\n"; - print " --filter (filter) : filter of input\n"; - print " --singlefilter : the input is the result of a single filter analysis"; print " --reduction (class) : Reduction class\n"; print " --verbose : Be extra verbose\n"; @@ -143,5 +141,5 @@ my $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $xcstack_id); if (! $ipprc->file_resolve($file)) { - print "\nfailed to resolve $file, is it a single-filter stack?\n"; + print "\nfailed to resolve $file, checking for a single-filter stack\n"; # input is from a single filter static sky run which used psphot instead of psphotStack # The file rule is different for psphot Index: /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcstack_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcstack_skycell.pl (revision 42662) +++ /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcstack_skycell.pl (revision 42663) @@ -120,4 +120,11 @@ my $stack_id = $input->{stack_id}; + my $good_frac = $input->{good_frac}; + my $Nsources = $input->{sources}; + + print "camera: $camera_name, stack_id: $stack_id, good_frac: $good_frac, Nsources: $Nsources\n"; + if ($good_frac < 1e-4) { my_escape("low good_frac ($good_frac) for $camera_name, stack_id $stack_id", $xcstack_id, 6000); } + if ($Nsources < 1) { my_escape("no sources for $camera_name, stack_id $stack_id", $xcstack_id, 6001); } + # check for the input files my $image = $ipprc->filename("PPSTACK.UNCONV", $input->{path_base}); # Image name @@ -131,8 +138,8 @@ my $have_sources = $ipprc->file_exists( $sources ); - &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_image; - &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_mask; - &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_weight; - &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_sources; + &my_die("Image $image does not exist for $camera_name, stack_id $stack_id", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_image; + &my_die("Mask $mask does not exist for $camera_name, stack_id $stack_id", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_mask; + &my_die("Weight $weight does not exist for $camera_name, stack_id $stack_id", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_weight; + &my_die("Sources $sources does not exist for $camera_name, stack_id $stack_id", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_sources; print $listFile "INPUT$num\tMETADATA\n"; @@ -148,5 +155,5 @@ close($listFile); -&my_die("input list contains less than two elements", $xcstack_id, $PS_EXIT_DATA_ERROR) unless scalar $num >= 2; +if ($num < 2) { &my_escape("not enough input stacks to make an xcstack", $xcstack_id, 6002); } print "I've loaded my inputs: $xcstack_id\n"; @@ -333,7 +340,35 @@ } $command .= " -dbname $dbname" if defined $dbname; + print "running in my_die: $command\n"; system ($command); } exit $exit_code; +} + +sub my_escape +{ + my $msg = shift; # Warning message on exit + my $xcstack_id = shift; # Xcstack identifier + my $quality = shift; # Exit code to add + + carp($msg); + + if (defined $xcstack_id and not $no_update) { + my $command = "$xcstacktool -xcstack_id $xcstack_id"; + if ($run_state eq 'new') { + $command .= " -addsumskyfile"; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -hostname $host" if defined $host; + $command .= " -quality $quality" if defined $quality; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + } else { + $command .= " -updatesumskyfile"; + $command .= " -set_quality $quality" if defined $quality; + } + $command .= " -dbname $dbname" if defined $dbname; + print "running in my_die: $command\n"; + system ($command); + } + exit 0; } Index: /branches/eam_branches/ipp-20230313/ippTasks/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20230313/ippTasks/Makefile.am (revision 42662) +++ /branches/eam_branches/ipp-20230313/ippTasks/Makefile.am (revision 42663) @@ -52,4 +52,6 @@ fpcamera.pro \ xcstack.pro \ + xcsky.pro \ + xccal.pro \ fullforce.pro Index: /branches/eam_branches/ipp-20230313/ippTasks/xccal.pro =================================================================== --- /branches/eam_branches/ipp-20230313/ippTasks/xccal.pro (revision 42663) +++ /branches/eam_branches/ipp-20230313/ippTasks/xccal.pro (revision 42663) @@ -0,0 +1,269 @@ +## xccal.pro : tasks for static sky calibration analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the +## xccal analysis. After a xccalRun entry (with +## associated xccal_id) is defined, psastro is performed (via script +## xccal.pl) (tasks in xccalRun). + +# test for required global variables +check.globals + +# skcal.pro should have a more restricted polling limit (to avoid stress with getstar) +# XXX: is this necessary anymore? +if ($?POLL_LIMIT_XCCAL == 0) set POLL_LIMIT_XCCAL = 64 + +macro set.xccal.poll + if ($0 != 2) + echo "USAGE:set.xccal.poll (value)" + break + end + + $POLL_LIMIT_XCCAL = $1 +end + +macro get.xccal.poll + echo "xccal poll limit: $POLL_LIMIT_XCCAL" +end + +### Initialise the books containing the tasks to do +book init xccalRun + +### Database lists +$xccal_DB = 0 +$xccal_revert_DB = 0 + +### Check status of xccal tasks +macro xccal.status + book listbook xccalRun +end + +### Reset xccal tasks +macro xccal.reset + book init xccalRun +end + +### Turn xccal tasks on +macro xccal.on + task xccal.load + active true + end + task xccal.run + active true + end +end + +### Turn xccal tasks off +macro xccal.off + task xccal.load + active false + end + task xccal.run + active false + end +end + +macro xccal.revert.on + task xccal.revert + active true + end +end + +macro xccal.revert.off + task xccal.revert + active false + end +end + +### Load tasks for xccal +### Tasks are loaded into xccalRun. +task xccal.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/xccal.log + + task.exec + if ($LABEL:n == 0) break + $run = xcskytool -pendingxccalrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xccal_DB + $run = $run -dbname $DB:$xccal_DB + $xccal_DB ++ + if ($xccal_DB >= $DB:n) set xccal_DB = 0 + end + add_poll_args run + add_poll_labels run + $run = $run -limit $POLL_LIMIT_XCCAL + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout xccalRun -key xccal_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook xccalRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup xccalRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the xccal analysis +### Tasks are taken from xccalRun. +task xccal.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages xccalRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in xccalRun (pantaskState == INIT) + book getpage xccalRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword xccalRun $pageName pantaskState RUN + book getword xccalRun $pageName xccal_id -var XCCAL_ID + book getword xccalRun $pageName xcstack_id -var XCSTACK_ID + book getword xccalRun $pageName tess_id -var TESS_DIR + book getword xccalRun $pageName skycell_id -var SKYCELL_ID + book getword xccalRun $pageName workdir -var WORKDIR_TEMPLATE + book getword xccalRun $pageName path_base -var PATH_BASE + book getword xccalRun $pageName reduction -var REDUCTION + book getword xccalRun $pageName state -var RUN_STATE + book getword xccalRun $pageName dbname -var DBNAME + + # move this above when we get camera into the db correctly: + # book getword xcskyResult $pageName camera -var CAMERA + $CAMERA = GPC1 + # $CAMERA = MEGACAM + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.stk.%s.xccal.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCSTACK_ID $XCCAL_ID + + stdout $LOGDIR/xccal.log + stderr $LOGDIR/xccal.log + + $run = xccal.pl --xccal_id $XCCAL_ID --outroot $outroot --redirect-output --camera $CAMERA --path_base $PATH_BASE --xcstack_id $XCSTACK_ID --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec .05 + command $run + end + + # default exit status + task.exit default + process_exit xccalRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword xccalRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword xccalRun $options:0 pantaskState TIMEOUT + end +end + +task xccal.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = xcskytool -revertxccal -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xccal_revert_DB + $run = $run -dbname $DB:$xccal_revert_DB + $xccal_revert_DB ++ + if ($xccal_revert_DB >= $DB:n) set xccal_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/eam_branches/ipp-20230313/ippTasks/xcsky.pro =================================================================== --- /branches/eam_branches/ipp-20230313/ippTasks/xcsky.pro (revision 42663) +++ /branches/eam_branches/ipp-20230313/ippTasks/xcsky.pro (revision 42663) @@ -0,0 +1,277 @@ +## xcsky.pro : tasks for xcsky photometry analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the +## xcsky photometry analysis. After a xcsky entry (with +## associated sky_id) is defined, psphotStack is performed (via script +## xcsky.pl) (tasks in xcskyResult). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init xcskyResult + +### Database lists +$xcsky_DB = 0 +$xcsky_revert_DB = 0 + +### Check status of xcsky tasks +macro xcsky.status + book listbook xcskyResult +end + +### Reset xcsky tasks +macro xcsky.reset + book init xcskyResult +end + +### Turn xcsky tasks on +macro xcsky.on + task xcsky.load + active true + end + task xcsky.run + active true + end + task xcsky.revert + active false + end +end + +### Turn xcsky tasks off +macro xcsky.off + task xcsky.load + active false + end + task xcsky.run + active false + end + task xcsky.revert + active false + end +end + +macro xcsky.revert.on + task xcsky.revert + active true + end +end + +macro xcsky.revert.off + task xcsky.revert + active false + end +end + +$RA_POLL_MAX = 0 + +macro set.ra.max + if ($0 != 2) + echo "USAGE: set.ra.max (ra_max_deg)" + break + end + $RA_POLL_MAX = $1 +end +macro get.ra.max + echo $RA_POLL_MAX +end + +### Load tasks for xcsky +### Tasks are loaded into xcskyResult. +task xcsky.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/xcsky.log + + task.exec + if ($LABEL:n == 0) break + $run = xcskytool -todo + if ($RA_POLL_MAX > 0) + $run = $run -ra_max $RA_POLL_MAX + end + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcsky_DB + $run = $run -dbname $DB:$xcsky_DB -limit 40 + $xcsky_DB ++ + if ($xcsky_DB >= $DB:n) set xcsky_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout xcskyResult -key xcsky_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook xcskyResult + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup xcskyResult + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the xcsky analysis +### Tasks are taken from xcskyResult. +task xcsky.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages xcskyResult -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new entries in xcskyResult (pantaskState == INIT) + book getpage xcskyResult 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword xcskyResult $pageName pantaskState RUN + book getword xcskyResult $pageName xcsky_id -var XCSKY_ID + book getword xcskyResult $pageName tess_id -var TESS_DIR + book getword xcskyResult $pageName skycell_id -var SKYCELL_ID + book getword xcskyResult $pageName workdir -var WORKDIR_TEMPLATE + book getword xcskyResult $pageName path_base -var PATH_BASE + book getword xcskyResult $pageName reduction -var REDUCTION + book getword xcskyResult $pageName dbname -var DBNAME + book getword xcskyResult $pageName state -var RUN_STATE + + # move this above when we get camera into the db correctly: + # book getword xcskyResult $pageName camera -var CAMERA + $CAMERA = GPC1 + # $CAMERA = MEGACAM + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + + if ("$PATH_BASE" == "NULL") + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.sky.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCSKY_ID + else + # run-state must be update use existing path_base + $outroot = $PATH_BASE + end + + stdout $LOGDIR/xcsky.log + stderr $LOGDIR/xcsky.log + + $run = xcsky.pl --xcsky_id $XCSKY_ID --run-state $RUN_STATE --outroot $outroot --redirect-output --camera $CAMERA --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are able to run the 'exec', use a short retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit xcskyResult $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword xcskyResult $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword xcskyResult $options:0 pantaskState TIMEOUT + end +end + +task xcsky.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it doesn't + # continue to occur). + $run = xcskytool -revert -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcsky_revert_DB + $run = $run -dbname $DB:$xcsky_revert_DB + $xcsky_revert_DB ++ + if ($xcsky_revert_DB >= $DB:n) set xcsky_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/eam_branches/ipp-20230313/ippTasks/xcstack.pro =================================================================== --- /branches/eam_branches/ipp-20230313/ippTasks/xcstack.pro (revision 42662) +++ /branches/eam_branches/ipp-20230313/ippTasks/xcstack.pro (revision 42663) @@ -91,4 +91,6 @@ task.exec + periods -exec $LOADEXEC + if ($LABEL:n == 0) break $run = xcstacktool -tosum @@ -106,4 +108,7 @@ # change the limit (the last one on the command line takes precedence) $run = $run -limit $POLL_LIMIT_XCSTACK + + # re-try quickly until we run out + periods -exec 0.05 command $run end Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c (revision 42662) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c (revision 42663) @@ -121,10 +121,10 @@ PXOPT_COPY_S64(config->args, whereMD, "-select_xcstack_id", "xcstackRun.xcstack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "xcstackRun.tess_id", "=="); PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "xcstackRun.skycell_id", "LIKE"); - PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "xcstackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "xcstackRun.filter", "LIKE"); PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "xcstackSumSkyfile.good_frac", ">="); pxAddLabelSearchArgs(config, whereMD, "-select_label", "xcstackRun.label", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "xcstackRun.data_group", "LIKE"); - pxAddLabelSearchArgs(config, whereMD, "-select_filter", "xcstackRun.filter", "LIKE"); if (!pxskycellAddWhere(config, whereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); @@ -1071,6 +1071,6 @@ PXOPT_COPY_S64(config->args, whereMD, "-select_xcsky_id", "xcskyRun.xcsky_id", "=="); PXOPT_COPY_S64(config->args, whereMD, "-select_xcstack_id", "xcstackRun.xcstack_id", "=="); - PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "xcstackRun.skycell_id", "=="); PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "xcstackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "xcstackRun.skycell_id", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "xcstackRun.filter", "LIKE"); PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "xcstackSumSkyfile.good_frac", ">="); Index: /branches/eam_branches/ipp-20230313/ippconfig/recipes/ppStack.config =================================================================== --- /branches/eam_branches/ipp-20230313/ippconfig/recipes/ppStack.config (revision 42662) +++ /branches/eam_branches/ipp-20230313/ippconfig/recipes/ppStack.config (revision 42663) @@ -253,5 +253,9 @@ SKIP.MISSING.INPUTS BOOL FALSE # If some input warp images are missing, make the stack anyway - SKIP.BG.SUB BOOL TRUE # Bypass background subtraction? + SKIP.BG.SUB BOOL TRUE # Bypass background subtraction? + + OUTPUT.NOCOMP BOOL FALSE # use filerules that make compressed outputs + OUTPUT.LOGFLUX BOOL TRUE # choose a log-scaling for output images + OUTPUT.DEEPEXP BOOL TRUE # allow exptime maps with dynamic range up to 2^16 (64k) seconds END Index: /branches/eam_branches/ipp-20230313/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/eam_branches/ipp-20230313/ippconfig/recipes/reductionClasses.mdc (revision 42662) +++ /branches/eam_branches/ipp-20230313/ippconfig/recipes/reductionClasses.mdc (revision 42663) @@ -2143,4 +2143,5 @@ PSVIDEOPHOT STR PSVIDEOPHOT XCSTACK_PPSTACK STR XCSTACK_UNIONS_DR4 - XCSTACK_PSPHOT STR STACKPHOT_SINGLE -END +# XCSTACK_PSPHOT STR STACKPHOT_SINGLE + XCSTACK_PSPHOT STR UNIONS_DR3 +END