Index: trunk/ippTasks/camera.pro =================================================================== --- trunk/ippTasks/camera.pro (revision 16174) +++ trunk/ippTasks/camera.pro (revision 16195) @@ -122,5 +122,5 @@ book getword camPendingExp $pageName exp_tag -var EXP_TAG book getword camPendingExp $pageName cam_id -var CAM_ID - book getword camPendingExp $pageName workdir -var WORKDIR + book getword camPendingExp $pageName workdir -var WORKDIR_TEMPLATE book getword camPendingExp $pageName dvodb -var DVODB book getword camPendingExp $pageName dbname -var DBNAME @@ -130,25 +130,26 @@ set.host.for.camera $camera FPA - # output log from filerule - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - sprintf outpath "%s/%s" $outpath $EXP_TAG - sprintf outroot "%s/%s.cm.%s" $outpath $EXP_TAG $CAM_ID + # raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + # out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + + ## generate output log based on filerule (convert the URI to a PATH) $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` if ("$logfile" == "") - $logfile = $outroot.log + echo "WARNING: logfile not defined in camera.pro:146" + break end stdout $logfile stderr $logfile - exec mkdir -p $outpath - $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA --outroot $outroot if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION Index: trunk/ippTasks/chip.pro =================================================================== --- trunk/ippTasks/chip.pro (revision 16174) +++ trunk/ippTasks/chip.pro (revision 16195) @@ -121,5 +121,5 @@ book getword chipPendingImfile $pageName exp_tag -var EXP_TAG book getword chipPendingImfile $pageName chip_id -var CHIP_ID - book getword chipPendingImfile $pageName workdir -var WORKDIR + book getword chipPendingImfile $pageName workdir -var WORKDIR_TEMPLATE book getword chipPendingImfile $pageName class_id -var CLASS_ID book getword chipPendingImfile $pageName uri -var URI @@ -130,25 +130,26 @@ set.host.for.camera $camera $class_id - ## generate output log based on filerule - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - sprintf outpath "%s/%s" $outpath $EXP_TAG - sprintf outroot "%s/%s.ch.%s" $outpath $EXP_TAG $CHIP_ID + # raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + # out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.ch.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID + + ## generate output log based on filerule (convert the URI to a PATH) $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` if ("$logfile" == "") - sprintf logfile "%s.%s.log" $outroot $CLASS_ID + echo "WARNING: logfile not defined in chip.pro:152" + break end stdout $logfile stderr $logfile - exec mkdir -p $outpath - $run = chip_imfile.pl --exp_id $EXP_ID --chip_id $CHIP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --uri $URI --camera $CAMERA - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + $run = chip_imfile.pl --exp_id $EXP_ID --chip_id $CHIP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --uri $URI --camera $CAMERA --outroot $outroot if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION Index: trunk/ippTasks/detrend.norm.pro =================================================================== --- trunk/ippTasks/detrend.norm.pro (revision 16174) +++ trunk/ippTasks/detrend.norm.pro (revision 16195) @@ -153,5 +153,5 @@ book getword detPendingNormStatImfile $pageName iteration -var ITERATION book getword detPendingNormStatImfile $pageName camera -var CAMERA - book getword detPendingNormStatImfile $pageName workdir -var WORKDIR + book getword detPendingNormStatImfile $pageName workdir -var WORKDIR_TEMPLATE book getword detPendingNormStatImfile $pageName dbname -var DBNAME @@ -159,19 +159,18 @@ set.host.for.camera $camera FPA - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID - $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.log + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + ## generate output log (NOT based on filerule?) + $logfile = "$outroot.log" + $logfile = `ipp_datapath.pl $logfile` + stdout $logfile stderr $logfile - exec mkdir -p $outpath - - $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + + $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --outroot $outroot add_standard_args run @@ -271,5 +270,5 @@ book getword detPendingNormImfile $pageName class_id -var CLASS_ID book getword detPendingNormImfile $pageName norm -var NORM - book getword detPendingNormImfile $pageName workdir -var WORKDIR + book getword detPendingNormImfile $pageName workdir -var WORKDIR_TEMPLATE book getword detPendingNormImfile $pageName dbname -var DBNAME @@ -277,20 +276,18 @@ set.host.for.camera $camera $class_id - # XXX use ipp_filename.pl to lookup output file names - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID - $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.$CLASS_ID.log + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this stack (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION $CLASS_ID + + ## generate output log (NOT based on filerule?) + $logfile = "$outroot.log" + $logfile = `ipp_datapath.pl $logfile` + stdout $logfile stderr $logfile - exec mkdir -p $outpath - - $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + + $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE --outroot $outroot add_standard_args run @@ -387,5 +384,5 @@ book getword detPendingNormExp $pageName det_type -var DET_TYPE book getword detPendingNormExp $pageName camera -var CAMERA - book getword detPendingNormExp $pageName workdir -var WORKDIR + book getword detPendingNormExp $pageName workdir -var WORKDIR_TEMPLATE book getword detPendingNormExp $pageName dbname -var DBNAME @@ -393,20 +390,18 @@ set.host.for.camera $camera FPA - # XXX add $WORKDIR/$LOG_DIR - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID - $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.log + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + ## generate output log (NOT based on filerule?) + $logfile = "$outroot.log" + $logfile = `ipp_datapath.pl $logfile` + stdout $logfile stderr $logfile - exec mkdir -p $outpath - - $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + + $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE --outroot $outroot add_standard_args run Index: trunk/ippTasks/detrend.process.pro =================================================================== --- trunk/ippTasks/detrend.process.pro (revision 16174) +++ trunk/ippTasks/detrend.process.pro (revision 16195) @@ -143,5 +143,5 @@ book getword detPendingProcessedImfile $pageName uri -var URI book getword detPendingProcessedImfile $pageName camera -var CAMERA - book getword detPendingProcessedImfile $pageName workdir -var WORKDIR + book getword detPendingProcessedImfile $pageName workdir -var WORKDIR_TEMPLATE book getword detPendingProcessedImfile $pageName dbname -var DBNAME book getword detPendingProcessedImfile $pageName reduction -var REDUCTION @@ -150,25 +150,21 @@ set.host.for.camera $camera $class_id - ## output log filename - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG - sprintf outroot "%s/%s.detproc.%s" $outpath $EXP_TAG $DET_ID + # see chip.pro for examples + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + ## generate output log based on filerule (convert the URI to a PATH) $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` if ("$logfile" == "") - sprintf logfile "%s.%s.log" $outroot $CLASS_ID + echo "WARNING: logfile not defined in detrend.process.pro:161" + break end stdout $logfile stderr $logfile - exec mkdir -p $outpath - - $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + + $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA --outroot $outroot if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION @@ -269,30 +265,30 @@ book getword detPendingProcessedExp $pageName camera -var CAMERA book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG - book getword detPendingProcessedExp $pageName workdir -var WORKDIR + book getword detPendingProcessedExp $pageName workdir -var WORKDIR_TEMPLATE book getword detPendingProcessedExp $pageName dbname -var DBNAME + book getword detPendingProcessedExp $pageName reduction -var REDUCTION # specify choice of local or remote host based on camera and chip (class_id) set.host.for.camera $camera FPA - # XXX push this into the filerules? - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG - sprintf outroot "%s/%s.detproc.%s" $outpath $EXP_TAG $DET_ID + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + ## generate output log based on filerule (convert the URI to a PATH) $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` if ("$logfile" == "") - $logfile = $outroot.log + echo "WARNING: logfile not defined in detrend.process.pro:282" + break end stdout $logfile stderr $logfile - exec mkdir -p $outpath - - $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR + + $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION end add_standard_args run Index: trunk/ippTasks/detrend.reject.pro =================================================================== --- trunk/ippTasks/detrend.reject.pro (revision 16174) +++ trunk/ippTasks/detrend.reject.pro (revision 16195) @@ -122,5 +122,5 @@ book getword detRejectExp $pageName mode -var MODE book getword detRejectExp $pageName camera -var CAMERA - book getword detRejectExp $pageName workdir -var WORKDIR + book getword detRejectExp $pageName workdir -var WORKDIR_TEMPLATE book getword detRejectExp $pageName dbname -var DBNAME @@ -128,19 +128,18 @@ set.host.for.camera $camera FPA - if ("$WORKDIR" == "NULL") - $outroot = `pwd` - else - $outroot = `ipp_datapath.pl $WORKDIR` - end - sprintf outroot "%s/%s.%s.%s" $outroot $CAMERA $DET_TYPE $DET_ID - sprintf logfile "%s/%s.%s.%s.%s.detreject.log" $outroot $CAMERA $DET_TYPE $DET_ID $ITERATION + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detreject.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = $outroot.log + $logfile = `ipp_datapath.pl $logfile` + stdout $logfile stderr $logfile - exec mkdir -p $outroot - $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA --outroot $outroot add_standard_args run Index: trunk/ippTasks/detrend.resid.pro =================================================================== --- trunk/ippTasks/detrend.resid.pro (revision 16174) +++ trunk/ippTasks/detrend.resid.pro (revision 16195) @@ -143,5 +143,5 @@ book getword detPendingResidImfile $pageName det_uri -var DET_URI book getword detPendingResidImfile $pageName camera -var CAMERA - book getword detPendingResidImfile $pageName workdir -var WORKDIR + book getword detPendingResidImfile $pageName workdir -var WORKDIR_TEMPLATE book getword detPendingResidImfile $pageName dbname -var DBNAME book getword detPendingResidImfile $pageName reduction -var REDUCTION @@ -150,25 +150,21 @@ set.host.for.camera $camera $class_id - # XXX use ipp_filename.pl to lookup output file names - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG - sprintf outroot "%s/%s.detresid.%s.%s" $outpath $EXP_TAG $DET_ID $ITERATION + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + ## generate output log based on filerule (convert the URI to a PATH) $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` if ("$logfile" == "") - sprintf logfile "%s.%s.log" $outroot $CLASS_ID + echo "WARNING: logfile not defined in detrend.resid.pro:161" + break end stdout $logfile stderr $logfile - exec mkdir -p $outpath - - $run = detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + + $run = detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE --outroot $outroot if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION @@ -272,5 +268,5 @@ book getword detPendingResidExp $pageName include -var INCLUDE book getword detPendingResidExp $pageName camera -var CAMERA - book getword detPendingResidExp $pageName workdir -var WORKDIR + book getword detPendingResidExp $pageName workdir -var WORKDIR_TEMPLATE book getword detPendingResidExp $pageName dbname -var DBNAME @@ -278,22 +274,21 @@ set.host.for.camera $camera FPA - # XXX use ipp_filename.pl to lookup output file names - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG - sprintf outroot "%s/%s.detresid.%s.%s" $outpath $EXP_TAG $DET_ID $ITERATION + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + ## generate output log based on filerule (convert the URI to a PATH) $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in detrend.process.pro:285" + break + end stdout $logfile stderr $logfile - exec mkdir -p $outpath - - $run = detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + + $run = detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot add_standard_args run Index: trunk/ippTasks/detrend.stack.pro =================================================================== --- trunk/ippTasks/detrend.stack.pro (revision 16174) +++ trunk/ippTasks/detrend.stack.pro (revision 16195) @@ -123,5 +123,5 @@ book getword detPendingStackedImfile $pageName class_id -var CLASS_ID book getword detPendingStackedImfile $pageName camera -var CAMERA - book getword detPendingStackedImfile $pageName workdir -var WORKDIR + book getword detPendingStackedImfile $pageName workdir -var WORKDIR_TEMPLATE book getword detPendingStackedImfile $pageName dbname -var DBNAME book getword detPendingStackedImfile $pageName reduction -var REDUCTION @@ -130,31 +130,28 @@ set.host.for.camera $camera $class_id - if ("$WORKDIR" == "NULL") - $outpath = `pwd` - else - $outpath = `ipp_datapath.pl $WORKDIR` - end - sprintf outpath "%s/%s.%s.%s" $outpath $CAMERA $DET_TYPE $DET_ID - sprintf outroot "%s/%s.%s.%s.%s" $outpath $CAMERA $DET_TYPE $DET_ID $ITERATION + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + ## generate output log based on filerule (convert the URI to a PATH) $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` if ("$logfile" == "") - sprintf logfile "%s.%s.log" $outroot $CLASS_ID + echo "WARNING: logfile not defined in detrend.stack.pro:141" + break end stdout $logfile stderr $logfile - exec mkdir -p $outpath - # save the pageName for future reference below - options $pageName - - $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA - if ("$WORKDIR" != "NULL") - $run = $run --workdir $WORKDIR - end + $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA --outroot $outroot if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION end add_standard_args run + + # save the pageName for future reference below + options $pageName # create the command line Index: trunk/ippTasks/notes.txt =================================================================== --- trunk/ippTasks/notes.txt (revision 16174) +++ trunk/ippTasks/notes.txt (revision 16195) @@ -1,4 +1,45 @@ -2008.01.18 + proposal to clean scripts/tasks up somewhat wrt names: + + * scripts do NOT use workdir to change the workdir in the db + * thus, what is supplied is actually the outputRoot + * require the output root for the scripts + * have the calling tasks construct the outputRoot + ** replace -workdir with -outroot + + some filename rules : + + * the db entry WORKDIR is the most abstract version: + path://GPC-@HOST@/20080130 + file://data/@HOST@.0/gpc1/20080130 + neb:///@HOST@/gpc1/20080130 + neb://gpc1/20080130 + XXX have nebulous distinguish between neb:///host/path and + neb://path? + + * the db WORKDIR entry is carried from step to step unless + over-ridden with a -queue command + + * psLib-based binaries (ppImage, etc) may be passed URIs, but not + files with the @HOST@ abstraction + + * IPP perl scripts may be passed URIs, but not files with the @HOST@ + abstraction + + * nebulous : convert neb:///@HOST@/path/name + + *** examples: + + * this tells ppImage to use the ipp004 version for both input and output: + ppImage -file neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01 + + * we already know the hostname at this point; include it in the call + chip_imfile.pl --uri neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits --workdir neb:///ipp004-v0/gpc1/20080130/ + + + + + +2008.01.21 I'm working on the summit.copy.pro interaction with nebulous (and @@ -25,16 +66,24 @@ host. + * specific output filenames + + In both ippTasks and ippScripts, there are references to files + which must correctly located on the host. + + * workdir choice of output directory. + + the + * dsget output target: dsget is called by pantasks in the task 'dsget'. currently, dsget accepts a --filename argument which specifies the output filename - to use. we have to tell t + to use. in the nebulous context, this is of the form + neb://top/next/stuff, without any host-specific information. To + target the file instance to a specific host, we pass dsget the + --volume (name) argument. The volumes refer to the specific + devices (potentially more than one per host), and the names can + then be chosen to include the host. - - * choice of output directory. - - the - - 2007.10.11 Index: trunk/ippTasks/pantasks.pro =================================================================== --- trunk/ippTasks/pantasks.pro (revision 16174) +++ trunk/ippTasks/pantasks.pro (revision 16195) @@ -342,28 +342,55 @@ macro set.workdir.by.camera + if ($0 != 6) + echo "USAGE: set.workdir.by.camera (camera) (class_id) (template) (default) (varname)" + echo " uses global variable NEBULOUS to determine if nebulous is active" + break + end + + local host default template camera classID varname + $camera = $1 + $classID = $2 + $template = $3 + $default = $4 + $varname = $5 + + # missing camera and/or chiphosts table results in host = NULL + book getword chiphosts $camera $classID -var host + + if ("$host" == "NULL") + strsub $template @HOST@ $default -var $$varname + else + strsub $template @HOST@ $host -var $$varname + end +end + +macro set.volume.by.camera if ($0 != 3) - echo "USAGE: set.workdir.by.camera (camera) (class_id)" - break - end + echo "USAGE: set.volume.by.camera (camera) (class_id) (template) (default) (varname)" + echo " uses global variable NEBULOUS to determine if nebulous is active" + break + end + + local host default template camera classID varname + $camera = $1 + $classID = $2 + $template = $3 + $default = $4 + $varname = $5 # missing camera and/or chiphosts table results in host = NULL - book getword chiphosts $1 $2 -var $host - - if ($NEBULOUS) - if ("$host" == "NULL") - $workdir = $workdir_default - $volume = "NULL" - return - end - strsub $workdir_template @HOST@ $host -var workdir - strsub $volume_template @HOST@ $host -var volume - return - end - - if ("$host" == "NULL") - $workdir = $workdir_default - return - end - strsub $workdir_template @HOST@ $host -var workdir + book getword chiphosts $camera $classID -var host + + if (not($NEBULOUS)) + $$varname = "NULL" + return + end + + if ("$host" == "NULL") + $$varname = $default + return + end + + strsub $template @HOST@ $host -var $$varname end Index: trunk/ippTasks/register.pro =================================================================== --- trunk/ippTasks/register.pro (revision 16174) +++ trunk/ippTasks/register.pro (revision 16195) @@ -137,10 +137,11 @@ book setword regPendingImfile $pageName pantaskState RUN - book getword regPendingImfile $pageName exp_id -var EXP_ID + book getword regPendingImfile $pageName exp_id -var EXP_ID book getword regPendingImfile $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingImfile $pageName tmp_camera -var TMP_CAMERA book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID - book getword regPendingImfile $pageName uri -var URI - book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE - book getword regPendingImfile $pageName dbname -var DBNAME + book getword regPendingImfile $pageName uri -var URI + book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingImfile $pageName dbname -var DBNAME # EXP_TAG is used to generate the unique, but human-readable, filenames @@ -148,26 +149,26 @@ # specify choice of remote host - set.host.for.camera $camera $class_id - - # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship - # this function uses workdir_template, workdir_default, volume_template, volume_default, - # it sets workdir and volume - ### this function needs to be improved: it is using fixed input and output var names - ### do we want these types of small outputs to go to a common filesystem? - set.workdir.by.camera $CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE - - ## generate output log based on filerule XXX put this in a function? - $outroot = `ipp_datapath.pl $WORKDIR` - sprintf outroot "%s/%s" $outroot $EXP_TAG - sprintf logfile "%s/%s.reg.log" $outroot $TMP_CLASS_ID + set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID + + # raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + # out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate output log based on filerule + sprintf logfile "%s/%s/%s.%s.reg.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID + $logfile = `ipp_datapath.pl $logfile` + stdout $logfile stderr $logfile - exec mkdir -p $outroot + + $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI + add_standard_args run # save the pageName for future reference below options $pageName - - $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI - add_standard_args run # create the command line @@ -254,32 +255,36 @@ book setword regPendingExp $pageName pantaskState RUN - book getword regPendingExp $pageName exp_id -var EXP_ID + book getword regPendingExp $pageName exp_id -var EXP_ID book getword regPendingExp $pageName tmp_exp_name -var TMP_EXP_NAME - book getword regPendingExp $pageName workdir -var WORKDIR - book getword regPendingExp $pageName dbname -var DBNAME + book getword regPendingExp $pageName tmp_camera -var TMP_CAMERA + book getword regPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingExp $pageName dbname -var DBNAME # EXP_TAG is used to generate the unique, but human-readable, filenames sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID - # specify choice of remote host:(need to choose based on chips) - if ($parallel) - host anyhost - else - host local - end + # specify choice of remote host + set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID + + # raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + # out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 ## generate output log based on filerule - $outroot = `ipp_datapath.pl $WORKDIR` - sprintf outroot "%s/%s" $outroot $EXP_TAG - sprintf logfile "%s/%s.reg.log" $outroot $EXP_TAG + sprintf logfile "%s/%s/%s.reg.log" $WORKDIR $EXP_TAG $EXP_TAG + $logfile = `ipp_datapath.pl $logfile` + stdout $logfile stderr $logfile - exec mkdir -p $outroot + + $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG + add_standard_args run # save the pageName for future reference below options $pageName - - $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG - add_standard_args run # create the command line Index: trunk/ippTasks/summit.copy.pro =================================================================== --- trunk/ippTasks/summit.copy.pro (revision 16174) +++ trunk/ippTasks/summit.copy.pro (revision 16195) @@ -18,11 +18,12 @@ # the templates are used if we have a class_id/host relationship; if none is found, the default values are used # XXX not sure how to handle the .N value if we need to use more than one +# XXX this stuff should all be placed in a 'site' config file if ($NEBULOUS) - $workdir_default = neb:// - $workdir_template = neb:// - $volume_default = ipp004-v0 - $volume_template = @HOST@-v0 + $default_host = ipp004 + $workdir_template = neb:///@HOST@-v0 + $default_volume = ipp004-v0 + $volume_template = else - $workdir_default = /data/ipp004.0/ + $default_host = ipp004 $workdir_template = /data/@HOST@.0/ end @@ -278,4 +279,7 @@ book getword pzPendingImfile $pageName camera -var $CAMERA + set.host.by.camera $CAMERA $CLASS_ID + host anyhost + # 2007-08-30T05:09:59Z substr $DATEOBS 0 4 YEAR @@ -284,21 +288,22 @@ # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship - # this function uses workdir_template, workdir_default, volume_template, volume_default, + # this function uses workdir_template, default_host, volume_template, volume_default, # it sets workdir and volume - set.workdir.by.camera $CAMERA $CLASS_ID + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + set.volume.by.camera $CAMERA $CLASS_ID $volume_template $default_volume volume # figure out filename - # XXXX this filename needs to use nebulous names - $FILENAME = $workdir/$CAMERA/$YEAR/$MONTH/$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits - $FILEROOT = $workdir_template/$CAMERA/$YEAR/$MONTH/$DAY + # XXX may need to use sprintf here + $FILENAME = $workdir_base/$CAMERA/$YEAR$MONTH$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits + $workdir = $workdir_template/$CAMERA/$YEAR$MONTH$DAY + # workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb://gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) book setword pzPendingImfile $pageName filename $FILENAME - - set.host.by.camera $CAMERA $CLASS_ID - host anyhost # store the pageName with this job options $pageName - options $FILEROOT + options $workdir $run = dsget --uri $URI --filename $FILENAME --bytes $BYTES @@ -330,6 +335,10 @@ # these are functions of (at least) EXP_TYPE and SURVEY_ID, both of which should come from the data store + # we saved the workdir so we can pass it to pztool + $workdir = $options:1 + # update the database with the success - $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $FILENAME -workdir $options:1 + # for the moment, let's not propagate these + $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $FILENAME -workdir $workdir -end_stage reg # tess_id, dvodb, workdir, end_stage