Index: /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in (revision 28980) @@ -103,4 +103,5 @@ $(DESTWWW)/chipProcessedImfile.php \ $(DESTWWW)/chipProcessedImfile_failure.php \ +$(DESTWWW)/failedChipProcessedImfile.php \ $(DESTWWW)/camSummary.php \ $(DESTWWW)/camStageExp.php \ @@ -109,4 +110,5 @@ $(DESTWWW)/camProcessedExp_Images.php \ $(DESTWWW)/camProcessedExp_failure.php \ +$(DESTWWW)/failedCamProcessedExp.php \ $(DESTWWW)/camIQstats.php \ $(DESTWWW)/camProcessedImfile.php \ @@ -118,4 +120,5 @@ $(DESTWWW)/fakeProcessedImfile.php \ $(DESTWWW)/fakeProcessedImfile_failure.php \ +$(DESTWWW)/failedFakeProcessedImfile.php \ $(DESTWWW)/pzPendingExp.php \ $(DESTWWW)/pzPendingImfile.php \ @@ -142,4 +145,5 @@ $(DESTWWW)/warpProcessedSkyfiles.php \ $(DESTWWW)/warpFailedSkyfiles.php \ +$(DESTWWW)/failedWarpSkyfiles.php \ $(DESTWWW)/diffSummary.php \ $(DESTWWW)/diffSummary_Images.php \ @@ -148,5 +152,5 @@ $(DESTWWW)/diffProcessedSkyfile.php \ $(DESTWWW)/diffProcessedSkyfile_Images.php \ -$(DESTWWW)/diffFailedSkyfile.php \ +$(DESTWWW)/failedDiffSkyfile.php \ $(DESTWWW)/stackRun.php \ $(DESTWWW)/stackSummary.php \ @@ -156,4 +160,5 @@ $(DESTWWW)/stackProcessedSkyfile_Images.php \ $(DESTWWW)/stackFailedSkyfile.php \ +$(DESTWWW)/failedStackSkyfile.php \ $(DESTWWW)/staticskyRun.php \ $(DESTWWW)/staticskySummary.php \ @@ -168,6 +173,9 @@ $(DESTWWW)/magicProcessedMasks_Images.php \ $(DESTWWW)/magicFailedTrees.php \ +$(DESTWWW)/failedMagicTrees.php \ $(DESTWWW)/magicFailedNodes.php \ +$(DESTWWW)/failedMagicNodes.php \ $(DESTWWW)/magicFailedMasks.php \ +$(DESTWWW)/failedMagicMasks.php \ $(DESTWWW)/distSummary.php \ $(DESTWWW)/distTarget.php \ Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedCamProcessedExp.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedCamProcessedExp.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedCamProcessedExp.d (revision 28980) @@ -0,0 +1,39 @@ +TABLE camRun, camProcessedExp, chipRun, rawExp +TITLE Camera Failed Exposures +FILE failedCamProcessedExp.php +MENU ipp.science.dat + +# the following WHERE clauses are added to all queries joined by AND +WHERE camRun.chip_id = chipRun.chip_id +WHERE chipRun.exp_id = rawExp.exp_id +WHERE camProcessedExp.cam_id = camRun.cam_id +WHERE camProcessedExp.fault != 0 + +# define the arguments supplied to the links below (if any) +ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id +ARGS ARG2 chipRun.chip_id=$chipRun.chip_id +ARGS ARG3 camRun.cam_id=$camRun.cam_id + +ARGS ARG4 camRun.cam_id=$camRun.cam_id +ARGS ARG4 camera=$rawExp.camera +ARGS ARG4 basename=$camProcessedExp.path_base + +OP OP1 ( $camProcessedExp.fault == 2 ? "cam.revert.on and cam.revert.off" : "camtool -revertprocessedexp -fault $camProcessedExp.fault -label $camRun.label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD rawExp.exp_name, 5, %s, Exp Name +FIELD rawExp.exp_id, 5, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD chipRun.chip_id, 5, %d, Chip ID, value, chipProcessedImfile.php, ARG2 +FIELD camRun.cam_id, 5, %d, Cam ID, value, camProcessedExp_failure.php, ARG3 +FIELD camRun.state, 5, %s, state, value, camProcessedImfile.php, ARG4 +FIELD camRun.label, 5, %s, label +FIELD camRun.data_group, 5, %s, data grp +FIELD camRun.dist_group, 5, %s, dist grp +FIELD camProcessedExp.fault, 2, %d, Fault +FIELD *, 50, %s, Command (Experimental), op=OP1 + +FIELD camProcessedExp.path_base, 5, %s, path_base, none +FIELD rawExp.exp_id, 5, %s, Exp ID, none + + +TAIL PHP insert_log ('LOG.IMFILE'); Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedChipProcessedImfile.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedChipProcessedImfile.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedChipProcessedImfile.d (revision 28980) @@ -0,0 +1,52 @@ +TABLE chipRun, rawExp, chipProcessedImfile +TITLE Chip Failed Imfiles +FILE failedChipProcessedImfile.php +MENU ipp.science.dat + +# limit this table to non-OBJECT types of images +WHERE chipRun.exp_id = rawExp.exp_id +WHERE chipProcessedImfile.exp_id = chipRun.exp_id +WHERE chipProcessedImfile.chip_id = chipRun.chip_id +WHERE chipProcessedImfile.fault != 0 + +ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id + +ARGS ARG2 chipRun.chip_id=$chipRun.chip_id + +ARGS ARG3 chipRun.chip_id=$chipRun.chip_id +ARGS ARG3 chipProcessedImfile.class_id=$chipProcessedImfile.class_id +ARGS ARG3 camera=$rawExp.camera +ARGS ARG3 basename=$chipProcessedImfile.path_base +ARGS ARG3 class=$chipProcessedImfile.class_id + +OP OP1 ( $chipProcessedImfile.fault == 2 ? "chip.revert.on and chip.revert.off" : "chiptool -revertprocessedimfile -fault $chipProcessedImfile.fault -label $chipRun.label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD rawExp.exp_name, 5, %s, Exp Name +FIELD chipProcessedImfile.class_id, 8, %s, Class ID +FIELD rawExp.exp_id, 5, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD chipRun.chip_id, 7, %d, Chip ID, value, chipProcessedImfile_failure.php, ARG2 +FIELD chipRun.state, 7, %s, State, value, chipProcessedImfile_failure.php, ARG3 +FIELD chipRun.label, 7, %s, Label +FIELD chipRun.data_group, 7, %s, data grp +FIELD chipRun.dist_group, 7, %s, dist grp +FIELD chipProcessedImfile.fault, 2, %d, Fault +FIELD *, 50, %s, Command Experimental, op=OP1 + +# FIELD rawExp.telescope, 10, %s, Telescope +# FIELD rawExp.camera, 10, %s, Camera +# FIELD rawExp.dateobs, 19, %T, Date/Time +# FIELD rawExp.ra, 8, %C, RA, op=OP1 +# FIELD rawExp.decl, 8, %C, DEC, op=OP2 +# FIELD rawExp.object, 8, %s, Object +# FIELD rawExp.filter, 10, %s, FILTER +# FIELD rawExp.exp_time, 5, %.2f, exp_time +# FIELD rawExp.airmass, 5, %.4f, airmass +# FIELD chipProcessedImfile.bg, 5, %.2f, backgnd +# FIELD chipProcessedImfile.bg_stdev, 5, %.2f, stdev +# FIELD chipProcessedImfile.n_stars, 5, %d, Nstars +# FIELD chipProcessedImfile.fwhm_major, 5, %.2f, FHWM +# FIELD chipProcessedImfile.path_base, 5, %s, path_base, none +# FIELD rawExp.comment, 65, %s, Comment + +TAIL PHP insert_log ('LOG.IMFILE'); Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedDiffSkyfile.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedDiffSkyfile.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedDiffSkyfile.d (revision 28980) @@ -0,0 +1,30 @@ +TABLE diffRun, diffSkyfile +TITLE Diff Failed Skyfiles +FILE failedDiffSkyfile.php +MENU ipp.stack.dat + +WHERE diffRun.diff_id = diffSkyfile.diff_id +WHERE diffSkyfile.fault != 0 + +ARGS ARG7 diffSkyfile.diff_id=$diffSkyfile.diff_id +ARGS ARG7 diffSkyfile.skycell_id=$diffSkyfile.skycell_id +ARGS ARG7 camera=GPC1 +ARGS ARG7 basename=$diffSkyfile.path_base + +OP OP1 ( $fault == 2 ? "diff.revert.on and diff.revert.off" : "difftool -revertdiffskyfile -fault $diffSkyfile.fault -label $diffRun.label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD diffSkyfile.diff_id, 5, %d, Diff ID +FIELD diffSkyfile.skycell_id, 5, %s, Skycell ID +FIELD diffRun.label, 5, %s, Label +FIELD diffRun.data_group, 5, %s, data grp +FIELD diffRun.dist_group, 5, %s, dist grp +# FIELD diffRun.filter, 5, %s, Filter +FIELD diffRun.tess_id, 5, %s, Tess ID +FIELD diffRun.state, 7, %s, State, value, diffProcessedSkyfile.php, ARG7 +FIELD diffSkyfile.fault, 5, %d, Fault +FIELD *, 50, %s, Command (Experimental), op=OP1 + +FIELD diffSkyfile.path_base, 5, %s, path_base, none + +TAIL PHP insert_log ('LOG.EXP'); Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedFakeProcessedImfile.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedFakeProcessedImfile.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedFakeProcessedImfile.d (revision 28980) @@ -0,0 +1,40 @@ +TABLE fakeRun, fakeProcessedImfile, camRun, chipRun, rawExp +TITLE fake Processed Imfile +FILE failedFakeProcessedImfile.php +MENU ipp.science.dat + +# limit this table to non-OBJECT types of images +WHERE fakeRun.cam_id = camRun.cam_id +WHERE camRun.chip_id = chipRun.chip_id +WHERE chipRun.exp_id = rawExp.exp_id +WHERE fakeProcessedImfile.fake_id = fakeRun.fake_id +WHERE fakeProcessedImfile.fault != 0 + +ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id +ARGS ARG2 chipRun.chip_id=$chipRun.chip_id +ARGS ARG3 camRun.cam_id=$camRun.cam_id +ARGS ARG4 fakeRun.fake_id=$fakeRun.fake_id + +ARGS ARG5 fakeRun.fake_id=fakeRun.fake_id +ARGS ARG5 fakeProcessedImfile.class_id=$fakeProcessedImfile.class_id +ARGS ARG5 camera=$rawExp.camera +ARGS ARG5 basename=$fakeProcessedImfile.path_base +ARGS ARG5 class=$fakeProcessedImfile.class_id + +OP OP1 ( $fakeProcessedImfile.fault == 2 ? "fake.revert.on and fake.revert.off" : "faketool -revertprocessedimfile -fault $fakeProcessedImfile.fault -label $fakeRun.label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD rawExp.exp_name, 10, %s, Exp Name +FIELD fakeProcessedImfile.class_id, 8, %s, Class ID +FIELD rawExp.exp_id, 5, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD chipRun.chip_id, 7, %d, Chip ID, value, chipProcessedImfile.php, ARG2 +FIELD camRun.cam_id, 7, %d, Cam ID, value, camProcessedExp.php, ARG3 +FIELD fakeRun.fake_id, 7, %d, Fake ID, value, fakeProcessedImfile_failure.php, ARG4 +FIELD fakeRun.state, 7, %s, State, value, fakeProcessedImfile_failure.php, ARG5 +FIELD fakeRun.label, 7, %s, Label +FIELD fakeProcessedImfile.fault, 2, %d, Fault +FIELD *, 50, %s, Command (Experimental/Not tested), op=OP1 + +FIELD fakeProcessedImfile.path_base, 5, %s, path_base, none + +TAIL PHP insert_log ('LOG.IMFILE'); Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicMasks.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicMasks.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicMasks.d (revision 28980) @@ -0,0 +1,21 @@ +TABLE magicRun, magicMask +TITLE Magic Failed Masks +FILE failedMagicMasks.php +MENU ipp.stack.dat + +WHERE magicRun.magic_id = magicMask.magic_id +WHERE magicMask.fault != 0 + +OP OP1 ( $fault == 2 ? "magic.revert.on and magic.revert.off" : "magictool -revertmask -fault $magicMask.fault -label $label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD magicRun.magic_id, 5, %d, Magic ID +FIELD magicRun.exp_id, 5, %d, Exp ID +FIELD magicRun.diff_id, 5, %d, Diff ID +FIELD magicRun.label, 5, %s, Label +FIELD magicRun.data_group, 5, %s, data grp +FIELD magicRun.inverse, 5, %d, Inverse +FIELD magicRun.state, 5, %s, State +FIELD magicMask.streaks, 5, %d, Streaks +FIELD magicMask.fault, 5, %d, Fault +FIELD *, 50, %s, Command (Experimental), op=OP1 Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicNodes.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicNodes.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicNodes.d (revision 28980) @@ -0,0 +1,24 @@ +TABLE magicRun, magicTree, magicNodeResult +TITLE Magic Failed Nodes +FILE failedMagicNodes.php +MENU ipp.stack.dat + +WHERE magicRun.magic_id = magicTree.magic_id +WHERE magicTree.magic_id = magicNodeResult.magic_id +WHERE magicTree.node = magicNodeResult.node +WHERE magicNodeResult.fault != 0 + +OP OP1 ( $magicNodeResult.fault == 2 ? "magic.revert.on and magic.revert.off" : "magictool -revertnode -fault $magicNodeResult.fault -label $magicRun.label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD magicRun.magic_id, 5, %d, Magic ID +FIELD magicRun.exp_id, 5, %d, Exp ID +FIELD magicRun.diff_id, 5, %d, Diff ID +FIELD magicRun.label, 5, %s, Label +FIELD magicRun.data_group, 5, %s, data grp +FIELD magicRun.inverse, 5, %d, Inverse +FIELD magicRun.state, 5, %s, State +FIELD magicTree.node, 5, %s, Node +FIELD magicTree.dep, 5, %s, Parent +FIELD magicNodeResult.fault, 5, %d, Fault +FIELD *, 50, %s, Command (Experimental), op=OP1 Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicTrees.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicTrees.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedMagicTrees.d (revision 28980) @@ -0,0 +1,19 @@ +TABLE magicRun +TITLE magic Failed Trees +FILE failedMagicTrees.php +MENU ipp.stack.dat + +WHERE fault != 0 + +OP OP1 ( $fault == 2 ? "magic.revert.on and magic.revert.off" : "magictool -reverttree -fault $fault -label $label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD magic_id, 5, %d, Magic ID +FIELD exp_id, 5, %d, Exp ID +FIELD diff_id, 5, %d, Diff ID +FIELD label, 5, %s, Label +FIELD data_group, 5, %s, Data Grp +FIELD inverse, 5, %d, Inverse +FIELD state, 5, %s, State +FIELD fault, 5, %d, Fault +FIELD *, 50, %s, Command (Experimental), op=OP1 Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedStackSkyfile.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedStackSkyfile.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedStackSkyfile.d (revision 28980) @@ -0,0 +1,31 @@ +TABLE stackRun, stackSumSkyfile +TITLE Stack Failed Skyfiles +FILE failedStackSkyfile.php +MENU ipp.stack.dat + +WHERE stackRun.stack_id = stackSumSkyfile.stack_id +WHERE stackSumSkyfile.fault != 0 + +ARGS ARG7 stackRun.stack_id=$stackRun.stack_id +ARGS ARG7 stackRun.skycell_id=$stackRun.skycell_id +ARGS ARG7 camera=GPC1 +ARGS ARG7 basename=$stackSumSkyfile.path_base + +OP OP1 ( $stackSumSkyfile.fault == 2 ? "stack.revert.on and stack.revert.off" : "stacktool -revertsumskyfile -fault $stackSumSkyfile.fault -label $stackRun.label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD stackRun.stack_id, 5, %s, Stack ID +FIELD stackRun.skycell_id, 5, %s, Skycell ID +FIELD stackRun.label, 5, %s, Label +FIELD stackRun.data_group, 5, %s, data grp +FIELD stackRun.dist_group, 5, %s, dist grp +FIELD stackRun.filter, 5, %s, Filter +FIELD stackRun.tess_id, 5, %s, Tess ID +FIELD stackRun.state, 7, %s, State, value, stackFailedSkyfile.php, ARG7 +FIELD stackSumSkyfile.fault, 2, %d, Fault + +FIELD *, 50, %s, Command you should try to run as the ipp user? (Experimental feature), op=OP1 + +FIELD stackSumSkyfile.path_base, 5, %s, path_base, none + +TAIL PHP insert_log ('LOG.EXP'); Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/failedWarpSkyfiles.d =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/def/failedWarpSkyfiles.d (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippMonitor/def/failedWarpSkyfiles.d (revision 28980) @@ -0,0 +1,42 @@ +TABLE warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp +TITLE Warp Failed Skyfiles +FILE failedWarpSkyfiles.php +MENU ipp.science.dat + +WHERE warpSkyfile.warp_id = warpRun.warp_id +WHERE warpRun.fake_id = fakeRun.fake_id +WHERE fakeRun.cam_id = camRun.cam_id +WHERE camRun.chip_id = chipRun.chip_id +WHERE chipRun.exp_id = rawExp.exp_id +WHERE warpSkyfile.fault != 0 + +ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id +ARGS ARG2 chipRun.chip_id=$chipRun.chip_id +ARGS ARG3 camRun.cam_id=$camRun.cam_id +ARGS ARG4 fakeRun.fake_id=$fakeRun.fake_id +ARGS ARG5 warpRun.warp_id=$warpRun.warp_id +ARGS ARG6 warpSkyfile.skycell_id=$warpSkyfile.skycell_id + +ARGS ARG7 warpRun.warp_id=$warpRun.warp_id +ARGS ARG7 warpSkyfile.skycell_id=$warpSkyfile.skycell_id +ARGS ARG7 camera=$rawExp.camera +ARGS ARG7 basename=$warpSkyfile.path_base + +OP OP1 ( $warpSkyfile.fault == 2 ? "warp.revert.on and warp.revert.off" : "warptool -revertwarped -fault $warpSkyfile.fault -label $warpRun.label -dbname ". $ID['proj'] ) + +# field size format name show link to extras +FIELD rawExp.exp_name, 5, %s, Exp Name +FIELD rawExp.exp_id, 5, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD chipRun.chip_id, 7, %d, Chip ID, value, chipProcessedImfile.php, ARG2 +FIELD camRun.cam_id, 7, %d, Cam ID, value, camProcessedExp.php, ARG3 +FIELD fakeRun.fake_id, 7, %d, Fake ID, value, fakeProcessedImfile.php, ARG4 +FIELD warpRun.warp_id, 7, %d, Warp ID, value, warpStageExp.php, ARG5 +FIELD warpSkyfile.skycell_id, 10, %s, Skycell ID, value, warpFailedSkyfiles.php, ARG6 +FIELD warpRun.state, 7, %s, State, value, warpFailedSkyfiles.php, ARG7 +FIELD warpRun.label, 7, %s, Label +FIELD warpSkyfile.fault, 2, %d, Fault +FIELD *, 50, %s, Command (Experimental), op=OP1 + +FIELD warpSkyfile.path_base, 5, %s, path_base, none + +TAIL PHP insert_log ('LOG.EXP'); Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_getplot.php =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_getplot.php (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_getplot.php (revision 28980) @@ -20,4 +20,7 @@ else if ($type=="h") $filePath = "/tmp/czarplot_".$label."_all_stages_".$type.".png"; +else if ($type=="s") +$filePath = "/tmp/czarplot_hosts_space.png"; + $file = fopen ($filePath, "r"); if ($file && !$debug) { Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_labels.php (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_labels.php (revision 28980) @@ -10,4 +10,6 @@ //$db = dbconnect($ID['proj']); $czardb = dbconnect("czardb"); // HACK to connect to czarDb +$lastUpdateTime = getLastUpdateTime($czardb); + $gpc1db = dbconnect($ID['proj']); @@ -22,5 +24,5 @@ else {$myMenu = "ipp.imfiles.dat";} -menu($myMenu, 'Czartool', 'ipp.css', $ID['link'], $ID['proj']); +menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']); $pass = $ID['pass']; @@ -31,18 +33,21 @@ $selectedLabel = $_GET[label]; $selectedServer = $_GET[server]; -$selectedRevertServer = $_GET[revertserver]; $selectedRevertStage = $_GET[revertstage]; $selectedRevertMode = $_GET[revertmode]; -if ($selectedLabel == "") $selectedLabel = "all_labels"; -if ($selectedStage == "") $selectedStage = "all_stages"; +if ($selectedLabel == "") { $selectedLabel = "all_stdscience_labels"; } +if ($selectedStage == "") { $selectedStage = "all_stages"; } + +echo 'Some documentation can be found here
'; // deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later -$currentRevertMode; -if ($selectedRevertStage != "" && $selectedRevertMode != "" && $selectedRevertServer != "" ) { - - exec("czartool_revert.pl -s $selectedRevertServer -t $selectedRevertStage -o $selectedRevertMode", $response, $status); +if ($selectedRevertStage != "" && $selectedRevertMode != "") { + + exec("czartool_revert.pl -t $selectedRevertStage -o $selectedRevertMode", $response, $status); + $currentRevertMode; if ($response[0] == "off") $currentRevertMode = 0; else if ($response[0] == "on") $currentRevertMode = 1; + + setRevertStatus($czardb, $selectedRevertStage, $currentRevertMode); } $debug = 0; @@ -51,4 +56,5 @@ $distLabels = getLabels($czardb, "distribution"); $pubLabels = getLabels($czardb, "publishing"); +$updateLabels = getLabels($czardb, "update"); if ($debug) { @@ -81,38 +87,53 @@ echo "\n"; echo "\n"; -$lastUpdateTime = getLastUpdateTime($czardb); echo "

Current status of IPP (any faults are shown in parentheses). NOTE: This data is good as of: $lastUpdateTime

"; echo "\n"; echo "\n"; + echo "\n"; echo "\n"; - - echo "\n"; - - echo "\n"; - echo "\n"; - -echo "
\n"; + echo "
"; + echo "		 \n"; - echo "
\n"; - echo "		 \n"; - createLabelsTable($pass, $proj, $czardb, $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $selectedRevertStage, $currentRevertMode); - echo "
\n"; - echo "\n"; - echo "\n"; - echo "\n"; - - echo "\n"; echo "\n"; -echo "
\n"; - echo "
\n"; - echo "		 \n"; - createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage); - echo " \n"; - $today = date("Y-m-d"); // 03.10.01 - showSummitData($gpc1db, $today); + createLabelsTable($pass, $proj, $czardb, "stdscience", $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage); echo "
\n"; -if ($selectedServer) showServerStatus($selectedServer); - + + echo "\n"; + echo "\n"; + echo "
"; + echo "\n"; + + echo " \n"; + createLabelsTable($pass, $proj, $czardb, "update", $updateLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage); + echo "\n"; + echo "\n"; + + echo "\n"; + echo "\n"; + echo "\n"; + echo "\n"; + echo "\n"; + + echo "\n"; + echo "\n"; + echo "
\n"; + echo "
"; + echo "		\n"; + createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage); + echo " \n"; + $today = date("Y-m-d"); + showSummitData($gpc1db, $today); + echo "
\n"; + + echo "\n"; + echo "\n"; + echo "\n"; + echo "\n"; + echo "\n"; + echo "
\n"; + echo "\n"; + if ($selectedServer) showServerStatus($selectedServer); + echo "
\n"; echo "\n"; echo "\n"; @@ -203,5 +224,5 @@ # ########################################################################### -function createLabelsTable($pass, $proj, $db, $stdsLabels, $distLabels, $pubLabels, $stages, $states, $selectedState, $selectedLabel, $selectedStage, $selectedRevertStage, $currentRevertMode) { +function createLabelsTable($pass, $proj, $db, $server, $labels, $distLabels, $pubLabels, $stages, $states, $selectedState, $selectedLabel, $selectedStage) { // set up table columns @@ -210,4 +231,6 @@ echo "\n"; + echo "

Current labels for $server server

"; + write_header_cell($class, ""); write_header_cell($class, ""); @@ -215,10 +238,6 @@ foreach ($stages as &$stage) { - $revertOnOff=array(); - if ($stage == "destreak" or $stage == "dist") $server = "distribution"; - else $server = "stdscience"; - if ($stage == $selectedRevertStage) $reverting = $currentRevertMode; - else $reverting = getRevertStatus($db, $stage); - $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&revertserver=" . $server . "&revertstage=" . $stage . "&revertmode="; + $reverting = getRevertStatus($db, $stage); + $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&revertmode="; if(!$reverting) {$label = "Start";$link = $link . "on";} if($reverting) {$label = "Stop";$link = $link . "off";} @@ -250,5 +269,5 @@ // write rows - foreach ($stdsLabels as &$stdsLabel) { + foreach ($labels as &$thisLabel) { $distributing = false; @@ -256,17 +275,17 @@ foreach ($distLabels as &$distLabel) { - if ($stdsLabel == $distLabel) { $distributing = true; break;} + if ($thisLabel == $distLabel) { $distributing = true; break;} } foreach ($pubLabels as &$pubLabel) { - if ($stdsLabel == $pubLabel) { $publishing = true; break;} + if ($thisLabel == $pubLabel) { $publishing = true; break;} } // create link to label summary page for each label - if ($stdsLabel == $selectedLabel) $link = ""; - else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel . "&stage=" . $selectedStage; + if ($thisLabel == $selectedLabel) $link = ""; + else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $thisLabel . "&stage=" . $selectedStage; echo "\n"; - write_table_cell($class, '%s', $link, $stdsLabel); + write_table_cell($class, '%s', $link, $thisLabel); write_table_cell($class, '%s', "", $distributing ? "yes" : "NO"); write_table_cell($class, '%s', "", $publishing ? "yes" : "NO"); @@ -275,90 +294,48 @@ $anyFaults = false; - $link = "chipProcessedImfile_failure.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $stdsLabel . "&chipRun.state=new"; - getStateAndFaults($db, $stdsLabel, $selectedState, "chip", $str, $anyFaults); + $link = "failedChipProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $thisLabel . "&chipRun.state=new"; + getStateAndFaults($db, $thisLabel, $selectedState, "chip", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&camRun.label=" . $stdsLabel . "&camRun.state=new"; - getStateAndFaults($db, $stdsLabel, $selectedState, "cam", $str, $anyFaults); + $link = "failedCamProcessedExp.php?pass=" . $pass . "&proj=" . $proj . "&camRun.label=" . $thisLabel . "&camRun.state=new"; + getStateAndFaults($db, $thisLabel, $selectedState, "cam", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); + $link = "failedFakeProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&fakeRun.label=" . $thisLabel . "&fakeRun.state=new"; + getStateAndFaults($db, $thisLabel, $selectedState, "fake", $str, $anyFaults); + write_table_cell($class, '%s', $anyFaults ? $link : "", $str); + + $link = "failedWarpSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.label=" . $thisLabel . "&warpRun.state=new"; + getStateAndFaults($db, $thisLabel, $selectedState, "warp", $str, $anyFaults); + write_table_cell($class, '%s', $anyFaults ? $link : "", $str); + + $link = "failedStackSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.label=" . $thisLabel . "&stackRun.state=new"; + getStateAndFaults($db, $thisLabel, $selectedState, "stack", $str, $anyFaults); + write_table_cell($class, '%s', $anyFaults ? $link : "", $str); + + $link = "failedDiffSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $thisLabel . "&diffRun.state=new"; + getStateAndFaults($db, $thisLabel, $selectedState, "diff", $str, $anyFaults); + write_table_cell($class, '%s', $anyFaults ? $link : "", $str); + $link = $defaultlink; - getStateAndFaults($db, $stdsLabel, $selectedState, "fake", $str, $anyFaults); + getStateAndFaults($db, $thisLabel, $selectedState, "magic", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.label=" . $stdsLabel . "&warpRun.state=new"; - getStateAndFaults($db, $stdsLabel, $selectedState, "warp", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.label=" . $stdsLabel . "&stackRun.state=new"; - getStateAndFaults($db, $stdsLabel, $selectedState, "stack", $str, $anyFaults); + $link = $defaultlink; + getStateAndFaults($db, $thisLabel, $selectedState, "magicDS", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $stdsLabel . "&diffRun.state=new"; - getStateAndFaults($db, $stdsLabel, $selectedState, "diff", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = $defaultlink; - getStateAndFaults($db, $stdsLabel, $selectedState, "magic", $str, $anyFaults); + getStateAndFaults($db, $thisLabel, $selectedState, "dist", $str, $anyFaults); write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - $link = $defaultlink; - getStateAndFaults($db, $stdsLabel, $selectedState, "magicDS", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = $defaultlink; - getStateAndFaults($db, $stdsLabel, $selectedState, "dist", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - echo "\n"; } - if ($selectedLabel == "all_labels") $link = ""; - else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_labels&stage=".$selectedStage; + if ($selectedLabel == "all_".$server."_labels") $link = ""; + else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_".$server."_labels&stage=".$selectedStage; echo "\n"; - write_table_cell($class, '%s', $link, "All labels:"); - write_table_cell($class, '%s', "", "-"); - write_table_cell($class, '%s', "", "-"); - - $str = ""; - $anyFaults = false; - - $link = "chipProcessedImfile_failure.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.state=new"; - getStateAndFaults($db, "all_labels", $selectedState, "chip", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&camRun.state=new"; - getStateAndFaults($db, "all_labels", $selectedState, "cam", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = $defaultlink; - getStateAndFaults($db, "all_labels", $selectedState, "fake", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.state=new"; - getStateAndFaults($db, "all_labels", $selectedState, "warp", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.state=new"; - getStateAndFaults($db, "all_labels", $selectedState, "stack", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.state=new"; - getStateAndFaults($db, "all_labels", $selectedState, "diff", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = $defaultlink; - getStateAndFaults($db, "all_labels", $selectedState, "magic", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = $defaultlink; - getStateAndFaults($db, "all_labels", $selectedState, "magicDS", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - - $link = $defaultlink; - getStateAndFaults($db, "all_labels", $selectedState, "dist", $str, $anyFaults); - write_table_cell($class, '%s', $anyFaults ? $link : "", $str); - + write_table_cell($class, '%s', $link, "All $server labels"); echo "\n"; @@ -368,4 +345,18 @@ ########################################################################### # +# Sets reverts status +# +########################################################################### +function setRevertStatus($db, $stage, $reverting) { + + $sql = "UPDATE reverts SET reverting = $reverting WHERE stage LIKE '$stage'"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} +} + +########################################################################### +# # Returns whether this stage is reverting or not # @@ -374,5 +365,6 @@ $anyFaults = false; - $sql = "SELECT reverting FROM $stage ORDER BY timestamp DESC LIMIT 1"; + + $sql = "SELECT reverting FROM reverts WHERE stage LIKE '$stage'"; if ($debug) {echo "$sql
";} Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat (revision 28980) @@ -25,4 +25,15 @@ menutop | menutop | plain |   | +menutop | menutop | link | New failed chip | failedChipProcessedImfile.php +menutop | menutop | link | New failed cam | failedCamProcessedExp.php +menutop | menutop | link | New failed fake | failedFakeProcessedImfile.php +menutop | menutop | link | New failed warp | failedWarpSkyfiles.php +menutop | menutop | link | New failed stack | failedStackSkyfile.php +menutop | menutop | link | New failed diff | failedDiffSkyfile.php +menutop | menutop | link | New failed magic tree | failedMagicTrees.php +menutop | menutop | link | New failed magic node | failedMagicNodes.php +menutop | menutop | link | New failed magic mask | failedMagicMasks.php + +menutop | menutop | plain |   | menulink | menuselect | link | Tables columns | columns_in_db.php menulink | menuselect | link | Clean /tmp directory | cleanTmpDirectory.php Index: /branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_revert.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_revert.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_revert.pl (revision 28980) @@ -5,19 +5,20 @@ use Getopt::Long; -my $server = undef; my $task = undef; my $onoff = undef; GetOptions ( - "server|s=s" => \$server, "task|t=s" => \$task, "onoff|o=s" => \$onoff); my $prob = 0; -if (!$server) {print "ERROR: need to define a server (-s)\n"; $prob=1;} if (!$task) {print "ERROR: need to define a task (-t)\n"; $prob=1;} if ($prob) {exit;} if ($task eq "cam") {$task = "camera";} + +my $server = undef; +if ($task eq "destreak" or $task eq "dist") {$server = "distribution";} +else {$server = "stdscience";} my $revert = undef; Index: /branches/eam_branches/ipp-20100621/ippMonitor/scripts/generate =================================================================== --- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/generate (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippMonitor/scripts/generate (revision 28980) @@ -648,5 +648,5 @@ foreach $word (@words) { $fword = &parse_fieldname ($word); - # print "field: $word -> $fword\n"; +# print "field: $word -> $fword\n"; push @outwords, $fword; if ($word =~ m|^\$|) { @@ -660,6 +660,6 @@ $opf{$name} = $outline; - # print "ops: $name : $ops{$name}\n"; - # print "opf: $name : $opf{$name}\n"; +# print "ops: $name : $ops{$name}\n"; +# print "opf: $name : $opf{$name}\n"; return 1; Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/checkOdmStatus.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/perl/checkOdmStatus.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/checkOdmStatus.pl (revision 28980) @@ -0,0 +1,182 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use LWP::UserAgent; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use XML::LibXML; +use File::Temp qw(tempfile); +use ippToPsps::IppToPspsDb; +use ippToPsps::Datastore; + +my $singleBatch = undef; +my $verbose = undef; +my $save_temps = undef; +my $fromTime = undef; +my $toTime = undef; +my $product = undef; + + +GetOptions( + 'batch|b=s' => \$singleBatch, + 'from|f=s' => \$fromTime, + 'to|t=s' => \$toTime, + 'product|p=s' => \$product, + 'verbose|v' => \$verbose, + 'save_temps|s' => \$save_temps + ); + +if (!defined $product) { + print "* OPTIONAL: a datastore product name -p \n"; +} +if (!defined $singleBatch) { + print "* OPTIONAL: a single batch -b (default = none)\n"; +} +if (!defined $fromTime) { + $fromTime = "2010-01-01"; + print "* OPTIONAL: from time -f (default = $fromTime)\n"; +} +if (!defined $toTime) { + $toTime = "2099-12-31"; + print "* OPTIONAL: to time -t (default = $toTime)\n"; +} +if (!defined $verbose) { + $verbose = 0; + print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; +} +if (!defined $save_temps) { + $save_temps = 0; + print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; +} + + +my $datastore = new ippToPsps::Datastore($product, 0, 0); +my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps); +my $odmUrl = "http://web01.psps.ifa.hawaii.edu/a01/OdmWebService/OdmWebService.asmx/GetBatchStatus"; +my $ua = LWP::UserAgent->new; +$ua->timeout(15); +$ua->env_proxy; + + +process(); + +####################################################################################### +# +# Loops through all processed exposures and checks against the ODM, then deletes if necessary +# +######################################################################################## +sub process { + + my $batches; + my $numOfBatches; + + if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getSingleBatch($singleBatch, \$batches);} + else { $numOfBatches = $ippToPspsDb->getBatchList(\$batches, $fromTime, $toTime);} + + if ($numOfBatches < 1) {return 0;} + + print "\n"; + printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n"); + printf("| Timestamp | Batch | Exposure ID | Loaded to ODM? | Merge worthy? | Merged? | Deleted? |\n"); + printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n"); + + # loop round batches + my $batch; + my $numChecked = 0; + foreach $batch ( @{$batches} ) { + my ($timestamp, $expId, $batchId, $surveyType, $deleted) = @{$batch}; + + if (checkBatch($timestamp, $expId, $batchId, $surveyType, $deleted)) {$numChecked++;} + + } + printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n"); + + printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numOfBatches); + +} + +######################################################################################y +# +# Check a single batch +# +######################################################################################## +sub checkBatch { + my ($timestamp, $expId, $batchId, $surveyType, $deleted) = @_; + + + my $batchFilter = sprintf("B%08d", $batchId); + my $statusFilter = "*"; + + my $response = $ua->post($odmUrl, + [batchNameFilter => $batchFilter, + statusFilter => $statusFilter, + fromFilter => "2010-01-01", + toFilter => "2099-01-01"] + ); + + # '200' is the 'OK' response + if ($response->code != 200) { + + print "Problem connecting to web service for '$batchFilter', HTTP response status: ".$response->status_line."\n"; + return; + } + # print( "HTTP response status: ".$response->content."\n" ); + + my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_odmXml.XXXX", UNLINK => !$save_temps); + print $tempFile $response->content; + close($tempFile); + + my $loadedToOdm = 0; + my $mergeWorthy = 0; + my $mergeCompleted = 0; + + parseXml($tempName, \$loadedToOdm, \$mergeWorthy, \$mergeCompleted); + + # delete from datastore + if(defined $product) { + + if (!$deleted && $loadedToOdm && $mergeWorthy) { + + $deleted = $datastore->remove($batchFilter); + if ($deleted) { + $ippToPspsDb->setBatchAsDeleted($batchId, $expId); + } + } + } + + printf( "| %18s | %11s | %10d | %6s | %6s | %6s | %5s |\n", + $timestamp, + $batchFilter, + $expId, + $loadedToOdm ? "yes" : "no", + $mergeWorthy ? "yes" : "no", + $mergeCompleted ? "yes" : "no", + $deleted ? "yes" : "no"); + + $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $mergeCompleted, $deleted); + +} + +######################################################################################y +# +# Parses ODM XML +# +######################################################################################## +sub parseXml { + my ($xmlFile, $loadedToOdm, $mergeWorthy, $mergeCompleted) = @_; + + my $parser = XML::LibXML->new; + my $doc = $parser->parse_file($xmlFile); + my $xc = XML::LibXML::XPathContext->new($doc); + $xc->registerNs('ArrayOfOdmBatchState', 'PanSTARRS.Services.OdmWebService'); + my $result = $xc->findvalue('//ArrayOfOdmBatchState:Message'); + ${$loadedToOdm} = 0; + ${$mergeWorthy} = 0; + ${$mergeCompleted} = 0; + + if ($result =~ m/LoadStarted/) { ${$loadedToOdm} = 1;} + if ($result =~ m/MergeWorthy/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1;} + if ($result =~ m/Merge[1-9]Completed/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1; ${$mergeCompleted} = 1;} +} + Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/convertPhotCodesToXml.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/perl/convertPhotCodesToXml.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/convertPhotCodesToXml.pl (revision 28980) @@ -0,0 +1,85 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use IPC::Cmd 0.36 qw( can_run run ); +use XML::Writer; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $inputPath = undef; +my $outputPath = "photcodes.xml"; + +# get user args +GetOptions( + 'input|i=s' => \$inputPath, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +# tell off user for not providing proper args +pod2usage( + -msg => "\n Required options:\n\n". + "--input \n". + -exitval => 3 + ) unless +defined $inputPath; + + + +my $output = new IO::File(">$outputPath"); +my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2); +$writer->xmlDecl('UTF-8'); + +$writer->startTag('table', "name" => "PhotoCal"); + +open (PHOTCODES, $inputPath); +my $photCode; +my $zeroPoint; +my $filter; +my $description; + +while () { + + chomp; + + if ($_ =~ m/\s+([0-9]+)\s+GPC1.*/) { + + my @columns = split(/\s+/, $_); + + my $filter = $columns[11]; + my $description = $columns[2]; + my $photCode = $columns[1]; + my $zeroPoint = $columns[4]; + my $extinction = $columns[5]; + + $writer->startTag('row', + "photoCalID" => $photCode, + "filterID" => $filter, + "photoCodeDesc" => $description, + "AB" => "0.0", # TODO + "zeropoint" => $zeroPoint, + "extinction" => $extinction, + "colorterm" => "0.0", # TODO + "colorExtn" => "0.0", # TODO + "orphanCalColor" => "0.0", # TODO + "orphanCalColorErr" => "0.0", # TODO + "startDate" => "54000."); + + $writer->endTag(); + + + + } +} + +$writer->endTag(); + + +close PHOTCODES; + +# finish up XML +#$writer->endTag(); +$writer->end(); + + Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Datastore.pm =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Datastore.pm (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Datastore.pm (revision 28980) @@ -0,0 +1,84 @@ +#!/usr/bin/perl -w + +package ippToPsps::Datastore; + +use warnings; +use strict; + +use File::Temp qw(tempfile); +use IPC::Cmd 0.36 qw( can_run run ); + +########################################################################### +# +# Constructor +# +########################################################################### +sub new { + + my $class = shift; + my $self = { + _product => shift, + _verbose => shift, + _save_temps => shift, + }; + + $self->{_dsreg} = can_run('dsreg') or (warn "* WARNING: Can't find 'dsreg' program"); + + bless $self, $class; + return $self; +} + +####################################################################################### +# +# Register item with datastore +# +######################################################################################## +sub register { + my ($self, $name, $path, $file, $type, $fileType) = @_; + + my ($tempFile, $tempName) = tempfile( "/tmp/Datastore_dsregList.XXXX", UNLINK => !$self->{_save_temps}); + + print $tempFile $file . '|||' . $fileType . "\n"; + + # build dsreg command command + my $command = "$self->{_dsreg}"; + $command .= " --add $name"; + $command .= " --copy"; + $command .= " --datapath $path"; + $command .= " --type $type"; + $command .= " --product $self->{_product}"; + $command .= " --list $tempName"; + + # run command + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $self->{_verbose}); + + if (!$success) { print "* Unable to publish $file to datastore\n" and return 0 }; + + print "* Successfully published $file to datastore\n"; + + close($tempFile); + + return 1; +} + +####################################################################################### +# +# Remove item from datastore +# +######################################################################################## +sub remove { + my ($self, $name) = @_; + + # build dsreg command command + my $command = "$self->{_dsreg}"; + $command .= " --del $name"; + $command .= " --product $self->{_product}"; + + # run command + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $self->{_verbose}); + + return $success; +} +1; Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Gpc1Db.pm =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Gpc1Db.pm (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Gpc1Db.pm (revision 28980) @@ -28,5 +28,4 @@ JOIN diffSkyfile USING (diff_id) WHERE rawExp.exp_id = $expId AND camRun.magicked - SQL @@ -37,5 +36,28 @@ ########################################################################### # -# Returns camera-stage smf files for this exposure +# Returns camera-stage smf file for this exposure as used in the provided DVO Db +# +########################################################################### +sub getCameraStageSmfForThisDvoDb { + my ($self, $dvoDb, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + return $query->fetchrow_array(); +} + +########################################################################### +# +# Returns latest processed camera-stage smf file for this exposure ID # ########################################################################### @@ -64,22 +86,20 @@ ########################################################################### # -# Returns a list of exposure IDs for this survey +# Returns a list of exposure IDs in a given DVO database # ########################################################################### -sub getExposureListFromSurvey { - my ($self, $survey) = @_; +sub getExposureListFromDvoDb { + my ($self, $dvoDb, $exposures) = @_; my $query = $self->{_db}->prepare(< 133887 @@ -93,27 +113,47 @@ $query->execute; - return $query->fetchall_arrayref(); + ${$exposures} = $query->fetchall_arrayref(); + my $numOfExposures = scalar @{${$exposures}}; + if ($numOfExposures > 0) { + + print "* Found $numOfExposures exposures in DVO Db '$dvoDb'\n"; + return 1; + } + + print "* No exposures found in DVO Db '$dvoDb'\n"; + return 0; } ########################################################################### # -# Returns info for this exposure ID +# Returns info for this exposure ID for given DVO Db # ########################################################################### -sub getExposureListFromExpId { - my ($self, $expId) = @_; +sub getSingleExposureFromDvoDb { + my ($self, $dvoDb, $expId, $exposures) = @_; my $query = $self->{_db}->prepare(<execute; - return $query->fetchall_arrayref(); + + ${$exposures} = $query->fetchall_arrayref(); + + if (scalar @{${$exposures}} > 0) { + + print "* Found exposure $expId in DVO Db '$dvoDb'\n"; + return 1; + } + + print "* Exposure $expId NOT found in DVO Db '$dvoDb'\n"; + return 0; } - - 1; Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPspsDb.pm =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 28980) @@ -8,4 +8,55 @@ use ippToPsps::MySQLDb; our @ISA = qw(ippToPsps::MySQLDb); # inherits from MySQLDb + + +########################################################################### +# +# Returns a list of batches that have been processed and loaded to datastore +# +########################################################################### +sub getBatchList { + my ($self, $exposures, $fromTime, $toTime) = @_; + + my $query = $self->{_db}->prepare(<= '$fromTime' + AND created <= '$toTime'; +SQL + + # TODO remove date restriction + $query->execute; + ${$exposures} = $query->fetchall_arrayref(); + my $count = scalar @{${$exposures}}; + + printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es"); + return $count; +} + +########################################################################### +# +# Returns a info about one particular batch +# +########################################################################### +sub getSingleBatch{ + my ($self, $batch_id, $exposures) = @_; + + my $query = $self->{_db}->prepare(<execute; + ${$exposures} = $query->fetchall_arrayref(); + my $count = scalar @{${$exposures}}; + + printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es"); + return $count; +} ####################################################################################### @@ -56,8 +107,9 @@ # ######################################################################################## -sub updateDb { - my ($self, $batchId, $expId, $processed, $published, $totalDetections) = @_; - -print "HJHJH '$batchId', '$expId', '$processed', '$published', '$totalDetections'\n"; +sub updateBatch { + my ($self, $batchId, $expId, $processed, $published, $totalDetections, $minObjId, $maxObjId) = @_; + + if (!$minObjId) {$minObjId = -1;} + if (!$maxObjId) {$maxObjId = -1;} if (!$totalDetections) {$totalDetections = -1;} @@ -65,5 +117,10 @@ my $query = $self->{_db}->prepare(< '2010-08-12' AND processed = 1; SQL @@ -102,6 +160,6 @@ # ######################################################################################## -sub getNewBatchId { - my ($self, $expId, $surveyType, $batchType) = @_; +sub createNewBatch { + my ($self, $expId, $surveyType, $batchType, $dvoDb, $datastoreProduct) = @_; my $query = $self->{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<execute; # TODO check response of these +} + +####################################################################################### +# +# Updates an existing database record with info from ODM +# +####################################################################################### +sub updateODMStatus { + my ($self,$batchId, $expId, $loadedToOdm, $mergeWorthy, $mergeCompleted) = @_; + + my $query = $self->{_db}->prepare(<execute; # TODO check response of these +} + + + ########################################################################### # @@ -142,10 +237,10 @@ my $currentRevision = -1; - my $latestRevision = 4; + my $latestRevision = 6; while ($currentRevision != $latestRevision) { $currentRevision = $self->getRevision(); - if ($self->{_verbose}) {print "* Current revision = $currentRevision\n";} + if ($self->{_verbose}) {print "* Current database revision = $currentRevision\n";} if ($currentRevision == 0) {$self->createRevision_1();} @@ -153,4 +248,6 @@ elsif ($currentRevision == 2) {$self->createRevision_3();} elsif ($currentRevision == 3) {$self->createRevision_4();} + elsif ($currentRevision == 4) {$self->createRevision_5();} + elsif ($currentRevision == 5) {$self->createRevision_6();} } } @@ -164,5 +261,5 @@ my ($self) = @_; - print "* Creating revision 1 of '$dbname'\n"; + print "* Creating revision 1 of '$self->{_dbName}'\n"; my $query = $self->{_db}->prepare(<execute; - my $query = $self->{_db}->prepare(<{_db}->prepare(<execute; - setRevision(1); + $self->setRevision(1); } @@ -203,5 +300,5 @@ my ($self) = @_; - print "* Creating revision 2 of '$dbname'\n"; + print "* Creating revision 2 of '$self->{_dbName}'\n"; my $query = $self->{_db}->prepare(<execute; - setRevision(2); + $self->setRevision(2); } @@ -222,5 +319,5 @@ my ($self) = @_; - print "* Creating revision 3 of '$dbname'\n"; + print "* Creating revision 3 of '$self->{_dbName}'\n"; my $query = $self->{_db}->prepare(<execute; - setRevision(3); + $self->setRevision(3); } @@ -241,5 +338,5 @@ my ($self) = @_; - print "* Creating revision 4 of '$dbname'\n"; + print "* Creating revision 4 of '$self->{_dbName}'\n"; my $query = $self->{_db}->prepare(<execute; - setRevision(4); + $self->setRevision(4); +} + +####################################################################################### +# +# Create revision 5 of the database +# +####################################################################################### +sub createRevision_5 { + my ($self) = @_; + + print "* Creating revision 5 of '$self->{_dbName}'\n"; + + my $query = $self->{_db}->prepare(<execute; + + $self->setRevision(5); +} + +####################################################################################### +# +# Create revision 6 of the database +# +####################################################################################### +sub createRevision_6 { + my ($self) = @_; + + print "* Creating revision 6 of '$self->{_dbName}'\n"; + + my $query = $self->{_db}->prepare(<execute; + + $self->setRevision(6); } Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl (revision 28980) @@ -6,21 +6,22 @@ use PS::IPP::Config 1.01 qw( :standard ); use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); use IPC::Cmd 0.36 qw( can_run run ); use File::Temp qw(tempfile); use XML::LibXML; +use File::Basename; # local classes use ippToPsps::Gpc1Db; use ippToPsps::IppToPspsDb; +use ippToPsps::Datastore; # globals my $camera = 'GPC1'; my $batchType = undef; -my $survey = undef; -my $dvodb = undef; +my $dvoLocation = undef; +my $dvoDb = undef; +my $fullDvoPath = undef; my $verbose = undef; my $save_temps = undef; -my $no_update = undef; my $output = undef; my $singleExpId = undef; @@ -34,11 +35,9 @@ 'output|o=s' => \$output, 'batch|b=s' => \$batchType, - 'dvodb|d=s' => \$dvodb, - 'survey|s=s' => \$survey, + 'dvo|d=s' => \$fullDvoPath, 'expid|e=s' => \$singleExpId, 'product|p=s' => \$datastoreProduct, 'verbose|v' => \$verbose, 'save_temps|t' => \$save_temps, - 'no-update|u' => \$no_update, 'force|f' => \$force, 'tarnzip|z' => \$dontTarNZip, @@ -49,41 +48,73 @@ print "* \n"; if (@ARGV) { - print "* UNKNKOWN: option: @ARGV\n"; $quit=1;} + $quit=1; + print "* UNKNKOWN: option @ARGV\n"; +} if (!defined $output) { - print "* REQUIRED: need to provide an output path: -o \n"; $quit=1;} + $quit=1; + print "* REQUIRED: need to provide an output path -o \n"; +} if (!defined $batchType) { - print "* REQUIRED: need to define a batch type: -b \n"; $quit=1;} -if (!defined $dvodb) { - print "* REQUIRED: need to provide a DVO Db path: -d \n"; $quit=1;} -if (!defined $survey and !defined $singleExpId) { - print "* REQUIRED: need to provide a survey type: -s ***OR***\n"; - print "* an exposure ID -e \n"; $quit=1;} + $quit=1; + print "* REQUIRED: need to define a batch type -b \n"; +} +if (!defined $fullDvoPath) { + $quit=1; + print "* REQUIRED: need to provide a DVO Db -d \n"; +} +if (!defined $singleExpId) { + + print "* OPTIONAL: a single exposure ID -e (default = none)\n"; +} if (!defined $datastoreProduct) { - print "* OPTIONAL: datastore product: -p \n";} + + print "* OPTIONAL: datastore product -p (default = none, i.e. data will not be published)\n"; +} if (!defined $verbose) { - print "* OPTIONAL: run in verbose mode: -v\n"; $verbose = 0;} + $verbose = 0; + print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; +} if (!defined $save_temps) { - print "* OPTIONAL: keep temp files: -t\n"; $save_temps = 0;} -if (!defined $no_update) { - print "* OPTIONAL: don't update database: -u\n"; $no_update = 0;} + $save_temps = 0; + print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; +} if (!defined $force) { - print "* OPTIONAL: force if already processed : -f\n"; $force = 0;} + $force = 0; + print "* OPTIONAL: force if already processed -f (default = $force)\n"; +} if (!defined $dontTarNZip) { - print "* OPTIONAL: don't tar and zip output : -z\n"; $dontTarNZip = 0;} - print "*\n*******************************************************************************\n"; + $dontTarNZip = 0; + print "* OPTIONAL: don't tar and zip output -z (default = $dontTarNZip)\n"; +} +print "*\n*******************************************************************************\n"; if ($quit) { exit; } + +# determine PSPS batch 'type' +my $pspsBatchType; +if ($batchType eq 'init') {$pspsBatchType = "IN";} +elsif ($batchType eq 'det') {$pspsBatchType = "P2";} +elsif ($batchType eq 'stack') {$pspsBatchType = "ST";} +elsif ($batchType eq 'diff') {$pspsBatchType = "OB";} +else {$pspsBatchType = "UNKNOWN";} + +# spilt full DVO path into Db and path +$fullDvoPath =~ s/\/$//; # strip off trailing '\' +($dvoDb, $dvoLocation) = fileparse($fullDvoPath); my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps); my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps); +my $datastore = undef; +if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);} # check we can run programs and get camera config my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR)); -my $dsreg = can_run('dsreg') or (warn "Can't find 'dsreg' program" and exit($PS_EXIT_CONFIG_ERROR)); my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); if ($batchType eq "init") {$initBatch = 1;} else {$initBatch = 0;} -process(); + +if (!process()) {print "* Finished unsuccessfully\n";} +else {print "* Finished successfully\n";} ####################################################################################### @@ -114,16 +145,22 @@ my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps); my $lastExpId = $ippToPspsDb->getLastExpId(); - my $rows = undef; + my $exposures; my $query; if ($initBatch) { - - my @row = (0, 'NULL', 'ThreePi'); - @{$rows} = (\@row); - } - elsif ($singleExpId) { $rows = $gpc1Db->getExposureListFromExpId($singleExpId); } - else { $rows = $gpc1Db->getExposureListFromSurvey($survey); } - -# TODO check if there are no exposures and give a warning + + my @exposure = (0, 'NULL', 'ThreePi'); + @{$exposures} = (\@exposure); + } + # get single exposure + elsif ($singleExpId) { + + if (!$gpc1Db->getSingleExposureFromDvoDb($dvoDb, $singleExpId, \$exposures)) {return 0;} + } + # get all exposures in this DVO Db + else { + + if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures)) {return 0;} + } my $batchId = 0; @@ -132,17 +169,12 @@ #my $batchId = -1; - my $row; - foreach $row ( @{$rows} ) { - my ($expId, $expName, $distGroup) = @{$row}; - # print "JHGHGHGHGH2 $expId, $expName, $distGroup\n"; - - # loop round exposures - #while (my @row = $query->fetchrow_array()) { - # my ($expId, $expName, $distGroup) = @row; + my $exposure; + foreach $exposure ( @{$exposures} ) { + my ($expId, $expName, $distGroup) = @{$exposure}; if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) { - - if ($force) {print "* Forcing....\n";} - else {next}; + + if ($force) {print "* Forcing....\n";} + else {next}; } @@ -150,5 +182,5 @@ if (!$surveyType) {next;} - $batchId = $ippToPspsDb->getNewBatchId($expId, $surveyType, $batchType); + $batchId = $ippToPspsDb->createNewBatch($expId, $distGroup, $pspsBatchType, $dvoDb, (defined $datastore) ? $datastoreProduct : "NONE"); # TODO quit here if no sensible batch ID @@ -156,6 +188,11 @@ # generate batch path from batch IDs my $batch = sprintf("B%08d", $batchId); - my $batchDir = sprintf("$output/$batch"); + my $dvoDir = "$output/$dvoDb"; + my $batchDir = "$dvoDir/$batch"; + + # make directories + unless(-d $dvoDir) {mkdir($dvoDir, 0777);} mkdir($batchDir, 0777); + $published = 0; @@ -171,11 +208,17 @@ if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) { + # tar n' zip if (!$dontTarNZip) { - my $tarball = tarAndZipBatch($output, $batch); - if ($tarball && $datastoreProduct && publishToDatastore($batch, $output, $tarball)) {$published = 1;} + my $tarball = tarAndZipBatch($dvoDir, $batch); + + # and publish + if ($tarball && defined $datastore ) { + + $published = $datastore->register($batch, $dvoDir, $tarball, "IPP_PSPS", "tgz"); + } } } - $ippToPspsDb->updateDb($batchId, $expId, 1, $published, $totalDetections); + $ippToPspsDb->updateBatch($batchId, $expId, 1, $published, $totalDetections, $minObjId, $maxObjId); } @@ -330,12 +373,4 @@ my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; - # determine batch 'type' - my $type; - if ($batchType eq 'init') {$type = "IN";} - elsif ($batchType eq 'det') {$type = "P2";} - elsif ($batchType eq 'stack') {$type = "ST";} - elsif ($batchType eq 'diff') {$type = "OB";} - else {$type = "UNKNOWN";} - # create XML file my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2); @@ -372,5 +407,5 @@ $writer->startTag('manifest', "name" => "$batch", - "type" => $type, + "type" => $pspsBatchType, "survey" => $pspsSurvey, "timestamp" => "$timeStamp", @@ -382,5 +417,5 @@ $writer->startTag('manifest', "name" => "$batch", - "type" => $type, + "type" => $pspsBatchType, "timestamp" => "$timeStamp"); } @@ -404,38 +439,4 @@ ####################################################################################### -# -# register new job with the datastore -# -######################################################################################## -sub publishToDatastore { - my ($batch, $path, $tarball) = @_; - - my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps); - - print $tempFile $tarball . '|||tgz' . "\n"; - - # build dsreg command command - my $command = "$dsreg"; - $command .= " --add $batch"; - $command .= " --copy"; - $command .= " --datapath $path"; - $command .= " --type IPP_PSPS"; - $command .= " --product $datastoreProduct"; - $command .= " --list $tempName"; - - # run command - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - if (!$success) { print "* Unable to publish $tarball to datastore\n" and return 0 }; - - print "* Successfully published $tarball to datastore\n"; - - close($tempFile); - - return 1; -} - -####################################################################################### # # runs ippToPsps to produce init batch @@ -457,5 +458,5 @@ my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; - my $nebPath = $gpc1Db->getCameraStageSmf($expId); + my $nebPath = $gpc1Db->getCameraStageSmfForThisDvoDb($dvoDb, $expId); if (!$nebPath) { return 0; } @@ -527,7 +528,7 @@ $command .= " -input $input"; $command .= " -output $output"; - $command .= " -D CATDIR $dvodb"; + $command .= " -D CATDIR $fullDvoPath"; $command .= " -config ../config"; # TODO - $command .= " -expid $expid"; + $command .= " -expid $expid"; $command .= " -expname $expName"; $command .= " -survey $surveyType"; Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/pspsSchema2xml.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/perl/pspsSchema2xml.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/pspsSchema2xml.pl (revision 28980) @@ -0,0 +1,337 @@ +#!/usr/bin/perl -w + +####################################################################################### +# +# Script that searches a dir containing PSPS schema files, finds those that comtain the +# tables on interest then parses them into an XML format. Also generates C-header files +# containing enums that detail table column names and numbers. +# +####################################################################################### + +use warnings; +use strict; +use IPC::Cmd 0.36 qw( can_run run ); +use XML::Writer; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $schemaPath = undef; +my $type = undef; + +# get user args +GetOptions( + 'schema|s=s' => \$schemaPath, + 'type|t=s' => \$type, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +# tell off user for not providing proper args +pod2usage( + -msg => "\n Required options:\n\n". + "--schema \n". + "--type \n". + -exitval => 3 + ) unless +defined $schemaPath and +defined $type; + +if ($type ne "init" && $type ne "det" && $type ne "diff" && $type ne "stack" && $type ne "object" ) { + + print "Don't understand type '$type'\n"; + die; +} + +my $enumsHeader = "ippToPsps".ucfirst($type)."Enums"; +open(OUT, ">".$enumsHeader.".h") or die("Error"); + +print OUT "#ifndef ".uc($enumsHeader)."_H\n"; +print OUT "#define ".uc($enumsHeader)."_H\n\n"; + + +my $output = new IO::File(">tables.xml"); +my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2); +$writer->xmlDecl('UTF-8'); +# $writer->doctype('manifest', "", "psps-manifest.dtd"); +$writer->startTag('tableDescriptions', "type" => "$type"); + +if ($type eq "init") {createInit();} +elsif ($type eq "det") {createDetections();} +elsif ($type eq "diff") {createDiffs();} +elsif ($type eq "stack") {createStacks();} +elsif ($type eq "object") {createObjects();} + +# finish up XML +$writer->endTag(); +$writer->end(); + +print OUT "\n#endif"; +close OUT; + + +####################################################################################### +# +# Finds the schema file containing the table name passed in +# +####################################################################################### +sub findSchemaFile { + my ($tableName) = @_; + + opendir(DIR, $schemaPath) or print "Cannot open '$schemaPath'\n" and return ""; + my @files= readdir(DIR); + closedir(DIR); + + foreach my $f (@files) { + + if ($f =~ m/.*\.sql$/) { + #print "....$f\n"; + + open FILE, "<$schemaPath/$f" or next; + + while () { + if ($_ =~ m/.*CREATE TABLE\s+dbo\.$tableName\s*\(/i) { + + close (FILE); + return "$schemaPath/$f"; + } + } + + close (FILE); + + } + } + + print "Could not find table '$tableName'\n"; + return ""; +} + +####################################################################################### +# +# Creates detection batch tables +# +####################################################################################### +sub createInit { + + parseTable("Filter"); + parseTable("FitModel"); + parseTable("PhotozRecipe"); + parseTable("Survey"); + parseTable("CameraConfig"); + parseTable("PhotoCal"); + parseTable("SkyCell"); + parseTable("ProjectionCell"); + parseTable("Region"); + parseTable("StackType"); + parseTable("ImageFlags"); + parseTable("DetectionFlags"); +} + +####################################################################################### +# +# Creates initialisation batch tables +# +####################################################################################### +sub createDetections { + + parseTable("FrameMeta"); + parseTable("ImageMeta"); + parseTable("Detection"); + parseTable("SkinnyObject"); + parseTable("ObjectCalColor"); +} + +####################################################################################### +# +# Creates difference batch tables +# +####################################################################################### +sub createDiffs { + + parseTable("StackMeta"); + parseTable("StackToImage"); + parseTable("StackLowSigDelta"); + parseTable("StackHighSigDelta"); + parseTable("ObjectCalColor"); +} + +####################################################################################### +# +# Creates stack batch tables +# +####################################################################################### +sub createStacks { + + parseTable("StackMeta"); + parseTable("StackDetection"); + parseTable("SkinnyObject"); + parseTable("StackOrphan"); + parseTable("ObjectCalColor"); +} + +####################################################################################### +# +# Creates object batch tables +# +####################################################################################### +sub createObjects { + + parseTable("Object"); +} + +####################################################################################### +# +# Parses a particular table from the SQL file and converts it to an XML description +# +####################################################################################### +sub parseTable { + my ($tableName, $newName) = @_; + + my $path = findSchemaFile($tableName); + + if ($path eq "") {return;} + + # sort out table name, either same as in schema or as defined + my $tableNameOut; + if ($newName) { $tableNameOut = $newName;} + else {$tableNameOut = $tableName} + + print OUT "\ntypedef enum {\n"; + $writer->startTag('table', "name" => $tableNameOut); + + open (SCHEMA, $path); + + my $reading = 0; + my $found = 0; + my $colNum = 0; + my $table; + + while () { + chomp; + + if ($_ =~ m/.*CREATE TABLE\s+dbo\.([a-zA-Z0-9.]+)\s*\(/i) { + + $table = $1; + + if ($table eq $tableName) { + + $reading = 1; + $found = 1; + $colNum = 0; + next; + } + } + + if($reading && $_ =~ m/^\s*\)\s*/) {$reading = 0;} + + if(!$reading) {next;} + + my $line = $_; + $line =~ s/[\s]*$//; + + if (!$line) {next;} + if (length($line) < 5) {next;} + if ($line =~ m/^\s*--/) {next;} + if ($line =~ m/\/\*/) {next;} + if ($line =~ m/\*\//) {next;} + + $colNum = processLine($line, $tableName, $colNum); + + } + + if (!$found) {print "Could not find table '$tableName'\n";} + $writer->endTag(); + print OUT "} ".$tableNameOut.";\n"; + + close SCHEMA; +} + +####################################################################################### +# +# Processes a line from the PSPS schema table description and converts to XML +# +####################################################################################### +sub processLine { + my ($line, $tableName, $colNum) = @_; + + my $name; + my $typeStr; + my $type; + my $comment; + my $default; + + # parse line + if ($line =~ m/\s*([a-zA-Z0-9()-\[\]]+)\s+(.*)\s*--\/(.*)/) { + + $name = $1; + $typeStr = $2; + $comment = $3; + + # neaten up the comment + $comment =~ s/<[\/]{0,1}column>//g; + $comment =~ s/^[\s]*//; + $comment =~ s/[\s]*$//; + if ($comment =~ m/(.*)/ ) {$comment = "$2 (unit = $1)"} + } + # no comment case + elsif ($line =~ m/\s*([a-zA-Z0-9()-\[\]]+)\s+(.*)/) { + + $name = $1; + $typeStr = $2; + $comment = "No comment"; + } + else { + + print "In '$tableName', can't process: '$line'\n"; + return $colNum + } + + my $ISBINARY = 0; + + # remove [] from name + $name =~ s/[\[\]]//g; + + # sort out type + if ($typeStr =~ m/([a-zA-Z0-9()-]*)[ \t]+.*/) { + + $type = $1; + $type =~ s/BIGINT/TLONGLONG/; + $type =~ s/SMALLINT/TSHORT/; + $type =~ s/TINYINT/TBYTE/; + $type =~ s/INT/TLONG/; + $type =~ s/FLOAT/TDOUBLE/; + $type =~ s/REAL/TFLOAT/; + $type =~ s/DATE/TSTRING/; + if ($type =~ m/^VARCHAR.*/) {$type = "TSTRING";} + if ($type =~ m/^VARBINARY.*/) {$type = "TSTRING"; $ISBINARY = 1;} + } + + # get default value + if ($typeStr =~ m/.*DEFAULT[ \t]*(.*)/i) { + + $default = $1; + $default =~ s/,//; + $default =~ s/[\(\)]//g; + $default =~ s/[\s]*$//; + $default =~ s/'//g; + if ($type ne "TSTRING" && !$default) {$default = "0";} + if ($ISBINARY && $default eq "0x") {$default = "";} + } + elsif ($type eq "TSTRING") {$default = " ";} + else {$default = 0;} + + if ($type eq "TSTRING" && $default eq "") {$default = " ";} + + $colNum++; + + print OUT " ".uc($tableName)."_".uc($name)." = ".$colNum.",\n"; + + $writer->startTag('column', + "name" => $name, + "type" => $type, + "default" => $default, + "comment" => $comment); + + $writer->endTag(); + + return $colNum; +} Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/removeFromDatastore.pl =================================================================== --- /branches/eam_branches/ipp-20100621/ippToPsps/perl/removeFromDatastore.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/removeFromDatastore.pl (revision 28980) @@ -0,0 +1,56 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use ippToPsps::Datastore; + +my $start = undef; +my $end = undef; +my $product = undef; + +GetOptions( + 'start|s=s' => \$start, + 'end|e=s' => \$end, + 'product|p=s' => \$product, + ); + +my $quit = 0; +print "\n*******************************************************************************\n"; +print "* \n"; +if (@ARGV) { + $quit=1; + print "* UNKNKOWN: option @ARGV\n"; +} +if (!defined $product) { + $quit=1; + print "* REQUIRED: a datastore product name -p \n"; +} +if (!defined $start) { + $quit=1; + print "* REQUIRED: starting batch number -s \n"; +} +if (!defined $end) { + $end=99999999999; + print "* OPTIONAL: ending batch number -e (default=$end)\n"; +} +print "*\n*******************************************************************************\n"; + +if ($quit) { exit; } + +my $datastore = new ippToPsps::Datastore($product, 0, 0); + +my $i; + +for ($i=$start; $i<$end+1; $i++) { + + my $name = sprintf("B%08d", $i); + if ($datastore->remove($name)) { + print "* Successfully removed $name from datastore\n"; + } + else { + print "* Unable to remove $name from datastore\n"; + } +} Index: /branches/eam_branches/ipp-20100621/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/Makefile.am (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/share/Makefile.am (revision 28980) @@ -6,4 +6,5 @@ addtool_checkminidvodbaddrun.sql \ addtool_donecleanup.sql \ + addtool_find_cam_id_dvo.sql \ addtool_find_cam_id.sql \ addtool_find_pendingexp.sql \ @@ -26,4 +27,6 @@ bgtool_definechip.sql \ bgtool_definewarp.sql \ + bgtool_listchip.sql \ + bgtool_listwarp.sql \ bgtool_revertchip.sql \ bgtool_revertwarp.sql \ @@ -150,4 +153,5 @@ disttool_definebyquery_fake.sql \ disttool_definebyquery_raw.sql \ + disttool_definebyquery_sky.sql \ disttool_definebyquery_stack.sql \ disttool_definebyquery_warp.sql \ @@ -163,4 +167,5 @@ disttool_pending_fake.sql \ disttool_pending_raw.sql \ + disttool_pending_sky.sql \ disttool_pending_stack.sql \ disttool_pending_warp.sql \ @@ -172,4 +177,5 @@ disttool_processedcomponent.sql \ disttool_queuercrun.sql \ + disttool_rerun_select.sql \ disttool_revertrcrun.sql \ disttool_revertrun.sql \ @@ -324,4 +330,7 @@ staticskytool_definebyquery_select.sql \ staticskytool_definebyquery_inputs.sql \ + staticskytool_export_run.sql \ + staticskytool_export_input.sql \ + staticskytool_export_result.sql \ staticskytool_inputs.sql \ staticskytool_todo.sql \ Index: /branches/eam_branches/ipp-20100621/ippTools/share/addtool_find_cam_id_dvo.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/addtool_find_cam_id_dvo.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/addtool_find_cam_id_dvo.sql (revision 28980) @@ -0,0 +1,10 @@ +SELECT camRun.* FROM camRun +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE camRun.state = 'full' + AND exp_id NOT IN (SELECT exp_id + FROM addRun + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + WHERE %s + ) Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_listchip.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_listchip.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_listchip.sql (revision 28980) @@ -0,0 +1,6 @@ +SELECT chip_bg_id, + class_id, + chipProcessedImfile.path_base +FROM chipBackgroundRun + JOIN chipBackgroundImfile USING(chip_bg_id) + JOIN chipProcessedImfile USING(chip_id, class_id) Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_listwarp.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_listwarp.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_listwarp.sql (revision 28980) @@ -0,0 +1,6 @@ +SELECT warp_bg_id, + skycell_id, + warpSkyfile.path_base +FROM warpBackgroundRun + JOIN warpBackgroundSkyfile USING(warp_bg_id) + JOIN warpSkyfile USING(warp_id, skycell_id) Index: /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_setimfiletoupdate.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_setimfiletoupdate.sql (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_setimfiletoupdate.sql (revision 28980) @@ -5,5 +5,6 @@ AND component = class_id) SET chipRun.state = 'update', - chipProcessedImfile.data_state = 'update' + chipProcessedImfile.data_state = 'update', + chipProcessedImfile.fault = 0 -- set hook %s WHERE chip_id = %lld Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_sky.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_sky.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_sky.sql (revision 28980) @@ -0,0 +1,27 @@ +SELECT DISTINCT + 'sky' as stage, + staticskyRun.sky_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'sky.$skycell_id.$stack_id' + CONCAT_WS('.', 'sky', stackRun.skycell_id, convert(staticskyRun.sky_id, CHAR)) as run_tag, + staticskyRun.label, + staticskyRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM staticskyRun +JOIN staticskyResult USING(sky_id) +JOIN staticskyInput USING(sky_id) +JOIN stackRun using(stack_id) +JOIN distTarget ON distTarget.stage = 'sky' + AND staticskyRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = sky_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL + AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned')) + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (staticskyResult.fault = 0 AND staticskyResult.quality = 0) Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_sky.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_sky.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_sky.sql (revision 28980) @@ -0,0 +1,50 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + stackRun.skycell_id AS component, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + staticskyResult.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + staticskyRun.state, + staticskyRun.state AS data_state, + staticskyResult.quality, + 1 AS no_magic, + 0 AS magicked +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN staticskyRun + ON stage_id = sky_id +JOIN staticskyResult + USING(sky_id) +JOIN staticskyInput + USING(sky_id) +JOIN stackRun USING(stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + USING (exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND stackRun.skycell_id = distComponent.component +WHERE + distRun.state = 'new' + AND distRun.stage = 'sky' + AND distComponent.dist_id IS NULL + AND (staticskyRun.state = 'full' OR (distRun.clean AND staticskyRun.state = 'cleaned')) + AND (staticskyResult.fault = 0 AND staticskyResult.quality = 0) Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_rerun_select.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_rerun_select.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_rerun_select.sql (revision 28980) @@ -0,0 +1,2 @@ +SELECT * FROM distRun +WHERE state = 'cleaned' Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_toadvance.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_toadvance.sql (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_toadvance.sql (revision 28980) @@ -240,4 +240,25 @@ AND distComponent.fault = 0 UNION +-- staticsky stage +-- NOTE this assumes that there is only one component per staticskyRun +-- (one skycell) +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN staticskyResult on stage_id = sky_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'sky' + AND distComponent.component IS NOT NULL + AND distComponent.fault = 0 +UNION -- SSdiff stage SELECT Index: /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_torevert_chip.sql (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_torevert_chip.sql (revision 28980) @@ -2,5 +2,5 @@ magic_ds_id, magicDSRun.state, - exp_id, + rawExp.exp_id, re_place, camera, @@ -19,7 +19,6 @@ JOIN camProcessedExp using(cam_id) JOIN chipRun ON (stage_id = chipRun.chip_id) - -- JOIN chipProcessedImfile ON (stage_id = chipProcessedImfile.chip_id AND component = class_id) - JOIN chipProcessedImfile USING(chip_id, exp_id) - JOIN rawExp using(exp_id) + JOIN chipProcessedImfile ON (stage_id = chipProcessedImfile.chip_id AND component = class_id) + JOIN rawExp ON chipRun.exp_id = rawExp.exp_id WHERE magicDSRun.stage = 'chip' AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) Index: /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql (revision 28980) @@ -987,6 +987,5 @@ projection_cell VARCHAR(64) NOT NULL, path_base VARCHAR(255) NOT NULL, - PRIMARY KEY(warp_id), - KEY(projection_cell), + PRIMARY KEY(warp_id, projection_cell), FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1086,6 +1085,5 @@ projection_cell VARCHAR(64) NOT NULL, path_base VARCHAR(255) NOT NULL, - PRIMARY KEY(sass_id), - KEY(projection_cell), + PRIMARY KEY(sass_id, projection_cell), FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1192,6 +1190,5 @@ projection_cell VARCHAR(64) NOT NULL, path_base VARCHAR(255) NOT NULL, - PRIMARY KEY(diff_id), - KEY(projection_cell), + PRIMARY KEY(diff_id, projection_cell), FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_input.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_input.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_input.sql (revision 28980) @@ -0,0 +1,1 @@ +SELECT staticskyInput.* FROM staticskyInput Index: /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_result.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_result.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_result.sql (revision 28980) @@ -0,0 +1,1 @@ +SELECT staticskyResult.* FROM staticskyResult Index: /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_run.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_run.sql (revision 28980) +++ /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_export_run.sql (revision 28980) @@ -0,0 +1,1 @@ +SELECT staticskyRun.* from staticskyRun Index: /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_result.sql =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_result.sql (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_result.sql (revision 28980) @@ -1,11 +1,12 @@ -SELECT +SELECT DISTINCT staticskyResult.*, staticskyRun.state, - staticskyRun.tess_id, - staticskyRun.skycell_id, - staticskyRun.filter, staticskyRun.workdir, staticskyRun.label, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter FROM staticskyRun -JOIN staticskyResult -USING(stack_id) +JOIN staticskyResult USING(sky_id) +JOIN staticskyInput USING(sky_id) +JOIN stackRun using(stack_id) Index: /branches/eam_branches/ipp-20100621/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/addtool.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/addtool.c (revision 28980) @@ -144,23 +144,25 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - - // find the cam_id of all the exposures that we want to queue up. - psString bare_query = pxDataGet("addtool_find_cam_id.sql"); - if (!bare_query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - psFree(where); - return false; - } - // prevent queueing an addRun if a given exposure has already been added to // the given dvo database psString dvodb_string = NULL; + psString bare_query = NULL; if (dvodb) { - // user supplied dvodb - psStringAppend(&dvodb_string, "(previous_dvodb = '%s')", dvodb); + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_cam_id_dvo.sql\n", dvodb); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_cam_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_cam_id.sql\n"); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_cam_id.sql"); // inherit dvodb from camRun, avoid matching NULL psStringAppend(&dvodb_string, "(camRun.dvodb IS NOT NULL AND previous_dvodb = camRun.dvodb)"); + } + if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; } // Take the bare query and add the dvodb selector Index: /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c (revision 28980) @@ -39,4 +39,5 @@ static bool advancechipMode(pxConfig *config); static bool revertchipMode(pxConfig *config); +static bool listchipMode(pxConfig *config); static bool definewarpMode(pxConfig *config); static bool updatewarpMode(pxConfig *config); @@ -47,4 +48,5 @@ static bool advancewarpMode(pxConfig *config); static bool revertwarpMode(pxConfig *config); +static bool listwarpMode(pxConfig *config); static bool tocleanchipMode(pxConfig *config); static bool cleanedchipMode(pxConfig *config); @@ -100,4 +102,5 @@ MODECASE(BGTOOL_MODE_ADVANCECHIP, advancechipMode); MODECASE(BGTOOL_MODE_REVERTCHIP, revertchipMode); + MODECASE(BGTOOL_MODE_LISTCHIP, listchipMode); MODECASE(BGTOOL_MODE_DEFINEWARP, definewarpMode); MODECASE(BGTOOL_MODE_UPDATEWARP, updatewarpMode); @@ -108,4 +111,5 @@ MODECASE(BGTOOL_MODE_ADVANCEWARP, advancewarpMode); MODECASE(BGTOOL_MODE_REVERTWARP, revertwarpMode); + MODECASE(BGTOOL_MODE_LISTWARP, listwarpMode); MODECASE(BGTOOL_MODE_TOCLEANCHIP, tocleanchipMode); MODECASE(BGTOOL_MODE_CLEANEDCHIP, cleanedchipMode); @@ -907,4 +911,67 @@ int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d chipBackgroundImfiles", numDeleted); + + return true; +} +static bool listchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + + // chip_bg_id is required + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("bgtool_listchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(whereStr); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); return true; @@ -1686,4 +1753,64 @@ return true; } +static bool listwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("bgtool_listwarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(whereStr); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} static bool tocleanwarpMode(pxConfig *config) Index: /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h (revision 28980) @@ -34,4 +34,5 @@ BGTOOL_MODE_ADVANCECHIP, BGTOOL_MODE_REVERTCHIP, + BGTOOL_MODE_LISTCHIP, // Warp stage BGTOOL_MODE_DEFINEWARP, @@ -43,4 +44,5 @@ BGTOOL_MODE_ADVANCEWARP, BGTOOL_MODE_REVERTWARP, + BGTOOL_MODE_LISTWARP, // Cleanups BGTOOL_MODE_TOCLEANCHIP, Index: /branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c (revision 28980) @@ -184,4 +184,11 @@ psMetadataAddS16(revertchipArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddBool(revertchipArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -listchip + psMetadata *listchipArgs = psMetadataAlloc(); + psMetadataAddS64(listchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id (required)", 0); + psMetadataAddStr(listchipArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); + psMetadataAddBool(listchipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(listchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -tocleanchip @@ -347,4 +354,11 @@ psMetadataAddBool(revertwarpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + // -listwarp + psMetadata *listwarpArgs = psMetadataAlloc(); + psMetadataAddS64(listwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id (required)", 0); + psMetadataAddStr(listwarpArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + psMetadataAddBool(listwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(listwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -tocleanwarp psMetadata *tocleanwarpArgs = psMetadataAlloc(); @@ -382,4 +396,5 @@ PXOPT_ADD_MODE("-advancechip", "", BGTOOL_MODE_ADVANCECHIP, advancechipArgs); PXOPT_ADD_MODE("-revertchip", "", BGTOOL_MODE_REVERTCHIP, revertchipArgs); + PXOPT_ADD_MODE("-listchip", "", BGTOOL_MODE_LISTCHIP, listchipArgs); PXOPT_ADD_MODE("-tocleanchip", "", BGTOOL_MODE_TOCLEANCHIP, tocleanchipArgs); PXOPT_ADD_MODE("-cleanedchip", "", BGTOOL_MODE_CLEANEDCHIP, cleanedchipArgs); @@ -395,4 +410,5 @@ PXOPT_ADD_MODE("-advancewarp", "", BGTOOL_MODE_ADVANCEWARP, advancewarpArgs); PXOPT_ADD_MODE("-revertwarp", "", BGTOOL_MODE_REVERTWARP, revertwarpArgs); + PXOPT_ADD_MODE("-listwarp", "", BGTOOL_MODE_LISTWARP, listwarpArgs); PXOPT_ADD_MODE("-tocleanwarp", "", BGTOOL_MODE_TOCLEANWARP, tocleanwarpArgs); PXOPT_ADD_MODE("-cleanedwarp", "", BGTOOL_MODE_CLEANEDWARP, cleanedwarpArgs); Index: /branches/eam_branches/ipp-20100621/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/disttool.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/disttool.c (revision 28980) @@ -34,4 +34,5 @@ static bool updaterunMode(pxConfig *config); static bool revertrunMode(pxConfig *config); +static bool rerunMode(pxConfig *config); static bool pendingcomponentMode(pxConfig *config); static bool addprocessedcomponentMode(pxConfig *config); @@ -83,4 +84,5 @@ MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode); + MODECASE(DISTTOOL_MODE_RERUN, rerunMode); MODECASE(DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentMode); MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode); @@ -327,4 +329,21 @@ // stack stage doesn't require magic no_magic = true; + } else if (!strcmp(stage, "sky")) { + magicRunType = "staticskyRun"; + query = pxDataGet("disttool_definebyquery_sky.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (staticskyRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (sticskyRun.dist_group = '%s')", dist_group); + } + // (static)sky stage doesn't require magic + no_magic = true; } else if (!strcmp(stage, "SSdiff")) { magicRunType = "diffRun"; @@ -587,4 +606,132 @@ } +static bool rerunMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "=="); + + // require data_group or dist_id to be supplied + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", true, false); + PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", false, false); + if (!data_group && !dist_id) { + psError(PXTOOLS_ERR_CONFIG, true, "data_group or dist_id is required"); + return false; + } + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + + psString label_hook = psStringCopy(""); + if (set_label) { + psStringAppend(&label_hook, ", label = '%s'", set_label); + } + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + + // It might be useful to be able to query by the parameters of the underlying runs + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("disttool_rerun_select.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + + if (pretend) { + if (!ippdbPrintMetadatas(stdout, output, "distRunsToUpdate", true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + query = "DELETE FROM distComponent where dist_id = %" PRId64; + + for (long i=0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 dist_id = psMetadataLookupS64(NULL, md, "dist_id"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, dist_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (!p_psDBRunQueryF(config->dbh, + "UPDATE distRun SET state = 'new', fault=0 %s WHERE dist_id =%" PRId64, + label_hook, dist_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + printf("re-running distRun %" PRId64 "\n", dist_id); + } + + psFree(output); + + return true; +} static bool revertrunMode(pxConfig *config) { Index: /branches/eam_branches/ipp-20100621/ippTools/src/disttool.h =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/disttool.h (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/disttool.h (revision 28980) @@ -28,4 +28,5 @@ DISTTOOL_MODE_UPDATERUN, DISTTOOL_MODE_REVERTRUN, + DISTTOOL_MODE_RERUN, DISTTOOL_MODE_PENDINGCOMPONENT, DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, Index: /branches/eam_branches/ipp-20100621/ippTools/src/disttoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/disttoolConfig.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/disttoolConfig.c (revision 28980) @@ -85,4 +85,16 @@ psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + // -rerun + psMetadata *rerunArgs = psMetadataAlloc(); + psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group (required)", NULL); + psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); + psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); + psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); + psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-set_label", 0, "define new label", NULL); + psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); + psMetadataAddU64(rerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -revertrun psMetadata *revertrunArgs = psMetadataAlloc(); @@ -327,10 +339,11 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-updaterun", "", DISTTOOL_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-revertrun", "", DISTTOOL_MODE_REVERTRUN, revertrunArgs); + PXOPT_ADD_MODE("-definebyquery", "define distribution run", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "update distRun", DISTTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-rerun", "reprocess a cleaned distribution run", DISTTOOL_MODE_RERUN, rerunArgs); + PXOPT_ADD_MODE("-revertrun", "revert distRun.faults", DISTTOOL_MODE_REVERTRUN, revertrunArgs); PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); - PXOPT_ADD_MODE("-revertcomponent", "", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); + PXOPT_ADD_MODE("-revertcomponent", "revert faulted components", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); PXOPT_ADD_MODE("-toadvance", "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs); @@ -340,5 +353,5 @@ PXOPT_ADD_MODE("-updatefileset", "", DISTTOOL_MODE_UPDATEFILESET, updatefilesetArgs); PXOPT_ADD_MODE("-listfilesets", "", DISTTOOL_MODE_LISTFILESETS, listfilesetsArgs); - PXOPT_ADD_MODE("-revertfileset", "", DISTTOOL_MODE_REVERTFILESET, revertfilesetArgs); + PXOPT_ADD_MODE("-revertfileset", "revert faulted filesets", DISTTOOL_MODE_REVERTFILESET, revertfilesetArgs); PXOPT_ADD_MODE("-queuercrun", "", DISTTOOL_MODE_QUEUERCRUN, queuercrunArgs); PXOPT_ADD_MODE("-updatercrun", "", DISTTOOL_MODE_UPDATERCRUN, updatercrunArgs); Index: /branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c (revision 28980) @@ -613,4 +613,5 @@ PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); Index: /branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c (revision 28980) @@ -120,4 +120,5 @@ psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage ID", 0); psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: /branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c (revision 28980) @@ -1220,4 +1220,6 @@ PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_STR(config->args, where, "-rlabel", "rlabel", "=="); pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); Index: /branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c (revision 28980) @@ -211,4 +211,6 @@ psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for dependent", NULL); psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define dep_id for dependent", 0); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-job_id", 0, "define job_id for dependent", 0); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-req_id", 0, "define eqp_id for dependent", 0); psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for pstampRequest", NULL); psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent", NULL); Index: /branches/eam_branches/ipp-20100621/ippTools/src/staticskytool.c =================================================================== --- /branches/eam_branches/ipp-20100621/ippTools/src/staticskytool.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/ippTools/src/staticskytool.c (revision 28980) @@ -727,5 +727,4 @@ bool exportrunMode(pxConfig *config) { -# if (0) typedef struct ExportTable { char tableName[80]; @@ -737,5 +736,5 @@ PS_ASSERT_PTR_NON_NULL(config, NULL); - PXOPT_LOOKUP_S64(det_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_S64(det_id, config->args, "-sky_id", true, false); PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); @@ -753,10 +752,10 @@ } psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); ExportTable tables [] = { - {"stackRun", "staticskytool_export_run.sql"}, - {"stackInputSkyfile", "staticskytool_export_input_skyfile.sql"}, - {"stackSumSkyfile", "staticskytool_export_sum_skyfile.sql"}, + {"staticskyRun", "staticskytool_export_run.sql"}, + {"staticskyInput", "staticskytool_export_input.sql"}, + {"staticskyResult", "staticskytool_export_result.sql"}, }; @@ -800,6 +799,6 @@ if (clean) { - if (!strcmp(tables[i].tableName, "stackRun")) { - if (!pxSetStateCleaned("stackRun", "state", output)) { + if (!strcmp(tables[i].tableName, "staticskyRun")) { + if (!pxSetStateCleaned("staticskyRun", "state", output)) { psFree(output); psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); @@ -821,5 +820,4 @@ fclose (f); -# endif return true; } Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_create =================================================================== --- /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_create (revision 28979) +++ /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_create (revision 28980) @@ -17,4 +17,5 @@ $query_id, $nostage, + $version, ); @@ -22,5 +23,6 @@ 'input|i=s' => \$input, 'output|o=s' => \$output, - 'query_id|q=s' => \$query_id, + 'query_id|q=s' => \$query_id, + 'version|v=s' => \$version, 'nostage' => \$nostage, ) or pod2usage( 2 ); @@ -30,5 +32,20 @@ -exitval => 3) unless defined $input and defined $output; -# The header kewords +# Read what we've been given +my $in; +if ($input eq '-') { + $in = \*STDIN; +} else { + open $in, "<$input" or die "cannot open $input for reading"; +} +my %colData; +my %headerData; +my $numRows = read_data_for_table($in,'\s+', \%colData, \%headerData); +if (!$numRows) { + print STDERR "no data in $input\n"; + exit 1; +} + +# The header keywords my $header = [ { name => 'QUERY_ID', @@ -62,16 +79,53 @@ value => undef }, - { name => 'STAGE', + { name => 'STAGE', writetype => TSTRING, comment => 'processing stage to examine', value => undef - } + } ]; -unless(defined($nostage)) { - push @{ $header }, { name => 'STAGE', - writetype => TSTRING, - comment => 'processing stage to examine', - value => undef - }; + +# Validate header. + +if (defined($query_id)) { + $headerData{QUERY_ID} = $query_id; +} +unless (exists($headerData{QUERY_ID})) { + die "No QUERY_ID specified for header."; +} + +unless (exists($headerData{EXTVER})) { + die "No EXTVER specified for header."; +} +if ($headerData{EXTVER} == 1) { + unless (exists($headerData{STAGE})) { + warn "No STAGE value specified in header. Assuming default value of 'diff'"; + $headerData{STAGE} = 'diff'; + } + foreach my $entry_ref (@{ $header }) { + my $name = $entry_ref->{name}; + unless (exists($headerData{$name})) { + die "Required header value $name not specified (try EXTVER=2?)."; + } + $entry_ref->{value} = $headerData{$name}; + } +} +elsif ($headerData{EXTVER} == 2) { + unless (exists($headerData{STAGE})) { + warn "No STAGE value specified in header. Assuming default value of 'diff'"; + $headerData{STAGE} = 'diff'; + } + my $tmp_header; + foreach my $entry_ref (@{ $header }) { + my $name = $entry_ref->{name}; + if (exists($headerData{$name})) { + $entry_ref->{value} = $headerData{$name}; + push @{ $tmp_header }, $entry_ref; + } + } + $header = $tmp_header; +} +else { + die "Unknown EXTVER = $headerData{EXTVER}."; } @@ -91,32 +145,51 @@ { name => 'MAG', type => 'D', writetype => TDOUBLE }, ]; - -my $in; -if ($input eq '-') { - $in = \*STDIN; -} else { - open $in, "<$input" or die "cannot open $input for reading"; -} - -my @colData; -foreach (@$columns) { - push @colData, []; -} - - -my $numRows = read_data_for_table($in,'\s+', \@colData, $header); -if (!$numRows) { - print STDERR "no data in $input\n"; - exit 1; -} - -# overwrite the QUERY_ID value from the input file with the command -# line argument - -if ($query_id) { - $header->[0]->{value} = $query_id; -} - -my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header); +my $columns_v2 = [ + { name => 'FPA_ID', type => '20A', writetype => TSTRING }, + { name => 'MJD-OBS', type => 'D', writetype => TDOUBLE }, + { name => 'FILTER', type => '20A', writetype => TSTRING }, + { name => 'OBSCODE', type => '20A', writetype => TSTRING }, + { name => 'STAGE', type => '20A', writetype => TSTRING } + ]; +# Validate the data. +if ($headerData{EXTVER} == 1) { + foreach my $entry_ref (@{ $columns }) { + my $name = $entry_ref->{name}; + unless (exists($colData{$name})) { + die "Required data column $name not found (try EXTVER=2?)."; + } + } +} +if ($headerData{EXTVER} == 2) { + my $tmp_columns; + foreach my $entry_ref (@{ $columns }) { + my $name = $entry_ref->{name}; + if (exists($colData{$name})) { + push @{ $tmp_columns }, $entry_ref; + } + } + foreach my $entry_ref (@{ $columns_v2 }) { + my $name = $entry_ref->{name}; + if (exists($colData{$name})) { + push @{ $tmp_columns }, $entry_ref; + } + } + $columns = $tmp_columns; +} + +# Construct the array of arrays +my @colDataAoA = (); +foreach my $col_def (@{ $columns }) { + my $name = $col_def->{name}; + push @colDataAoA, $colData{$name}; +} + +# foreach (@$columns) { +# push @colData, []; +# } + + + +my $status = make_fits_table($output, EXTNAME, $numRows, \@colDataAoA, $columns, $header); exit $status; @@ -230,45 +303,49 @@ my $line_num = 0; - - # read data for header if any data is expected - if ($header) { - my $nhead = @$header; - while (my $line = <$in>) { - $line_num++; - chomp $line; - next if !$line; # skip blank lines - next if ($line =~ /^#/); # skip comment lines - my @vals = split /$sep/, $line; - my $nvals = @vals; - die "number of header columns in input $nvals does not equal expected number of header words $nhead" - if (@vals != @$header); - - for (my $i=0; $i < @$header; $i++) { - $header->[$i]->{value} = $vals[$i]; - } - - last; # only one header line - } - } - - my $row_num = 0; - my $ncols = @$colData; - while (my $line = <$in>) { - chomp $line; - $line_num++; - next if !$line; # skip blank lines - next if ($line =~ /^#/); # skip comment lines - - my @vals = split /$sep/, $line; - my $nvals = @vals; - die "number of columns $nvals in input does not equal expected number of header " - . " words $ncols on line $line_num" if ($nvals != $ncols); - - for (my $col = 0; $col < @$colData; $col++) { - $colData->[$col]->[$row_num] = $vals[$col]; - } - $row_num++; - } - + my @keywords = (); + my $row_num; + while (<$in>) { + chomp; + if ($line_num == 0) { + # Parse header information keywords + $_ =~ s/#//g; + @keywords = split /\s+/; + if ($keywords[0] eq '') { + shift(@keywords); + } + } + elsif ($line_num == 1) { + # Parse header information values + my @values = split /\s+/; + if ($#values != $#keywords) { + die "Number of header columns in input does not equal expected number of header words"; + } + for (my $i = 0; $i <= $#values; $i++) { + $header->{$keywords[$i]} = $values[$i]; + } + } + elsif ($line_num == 2) { + # Parse table information keywords, dumping old keywords + $_ =~ s/#//g; + @keywords = split /\s+/; + if ($keywords[0] eq '') { + shift(@keywords); + } + } + else { + # Parse table information values + unless ($_ =~ /^#/) { + my @values = split /\s+/; + if ($#values != $#keywords) { + die "Number of header columns in input does not equal expected number of header words"; + } + for (my $i = 0; $i <= $#values; $i++) { + push @{ $colData{$keywords[$i]} }, $values[$i]; + $row_num = $#{ $colData{$keywords[$i] } } + 1; + } + } + } + $line_num++; + } # we return the number of rows read return $row_num; Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read =================================================================== --- /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read (revision 28979) +++ /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read (revision 28980) @@ -20,4 +20,5 @@ use Math::Trig; use Data::Dumper; +use IPC::Cmd 0.36 qw( can_run run ); use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for table @@ -31,9 +32,15 @@ $output, # Name of output table $save_temps, # Save temporary files? + $dbname, # needed to do camtool lookups. ); + +my $regtool = can_run('regtool') or (die "Can't find regtool"); +my $camtool = can_run('camtool') or (die "Can't find regtool"); + GetOptions( 'input|i=s' => \$input, 'output|o=s' => \$output, + 'dbname=s' => \$dbname, 'nolabel|l' => \$no_print_label, 'noheader|h' => \$no_print_header, @@ -65,60 +72,23 @@ # The keywords found in the header -# my $header = { -# 'EXTVER' => { name => 'EXTVER', -# writetype => TSTRING, -# comment => 'Extension version', -# value => undef -# }, -# 'EXTNAME' => { name => 'EXTNAME', -# writetype => TSTRING, -# comment => 'name of this binary table extension', -# value => undef -# }, -# 'QUERY_ID' => { name => 'QUERY_ID', -# writetype => TSTRING, -# comment => 'MOPS Query ID for this batch query', -# value => undef -# }, -# 'FPA_ID' => { name => 'FPA_ID', -# writetype => TSTRING, -# comment => 'orginal FPA_ID used at ingest', -# value => undef -# }, -# 'MJD_OBS' => { name => 'MJD-OBS', -# writetype => TDOUBLE, -# comment => 'starting time of the exposure, MJD', -# value => undef -# }, -# 'FILTER' => { name => 'FILTER', -# writetype => TSTRING, -# comment => 'effective filter use for the exposure', -# value => undef -# }, -# 'OBSCODE' => { name => 'OBSCODE', -# writetype => TSTRING, -# comment => 'site identifier (MPC observatory code)', -# value => undef -# }, -# 'STAGE' => { -# name => 'STAGE', -# writetype => TSTRING, -# comment => 'processing stage to examine', -# value => undef -# } -# }; my $parse_error = 0; -# Parse the header to determine what we expect to find. +# Parse the header to determine what we have to work with. +# Clean up empty space and remove unneeded single quotes. foreach my $header_key (keys %{ $inHeader }) { $inHeader->{$header_key} =~ s/\s+//g; $inHeader->{$header_key} =~ s/\'//g; } -my ($EXTVER,$headerEXTNAME,$QUERY_ID,$FPA_ID,$MJD_OBS,$FILTER,$OBSCODE,$STAGE) = - ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},$inHeader->{FPA_ID}, - $inHeader->{'MJD-OBS'},$inHeader->{FILTER},$inHeader->{OBSCODE},$inHeader->{STAGE}); +my ($EXTVER,$headerEXTNAME,$QUERY_ID, + $FPA_ID,$MJD_OBS,$FILTER, + $OBSCODE,$STAGE) = + ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID}, + $inHeader->{FPA_ID},$inHeader->{'MJD-OBS'},$inHeader->{FILTER}, + $inHeader->{OBSCODE},$inHeader->{STAGE}); + unless(defined($EXTVER) && defined($headerEXTNAME) && (($EXTVER == 1)||($EXTVER == 2)) && - ($headerEXTNAME eq EXTNAME)) { + ($headerEXTNAME eq EXTNAME)&& + (defined($QUERY_ID))) { $parse_error = 1; } @@ -143,5 +113,5 @@ # apparent magnitude { name => 'MAG', type => 'D', writetype => TDOUBLE, version => 1 }, - # v2 query_id: MOPS query ID for this batch query + # v2 query_id: needs to be here. { name => 'QUERY_ID', type => '20A', writetype => TSTRING, version => 2, default => $QUERY_ID }, # v2 fpa_id: original FPA_ID used at ingest @@ -172,14 +142,109 @@ if ($col->{version} > $EXTVER) { - @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows); + @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1); next; } - $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status); - $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status); - $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) - and check_fitsio($status); - $colData{$col->{name}} = $col_data; - if ($col->{name} eq 'MJD-OBS') { - print @{ $col_data } . "\n"; + else { + if (defined($col->{default})) { + @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1); + next; + } + } + + $inFits->get_colnum(0, $col->{name}, $col_num, $status); + if ($status == 0) { + $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status); + $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) + and check_fitsio($status); + $colData{$col->{name}} = $col_data; + if ($col->{name} eq 'QUERY_ID') { + print @{ $col_data } . "\n"; + } + + } + elsif ($status == 219) { + @{ $colData{$col->{name}} } = map { "Not Set" } (0 .. $numRows - 1); + $status = 0; + } +} + +# Set things that aren't set. +for (my $i = 0; $i < $numRows; $i++) { +# Simple stuff first. + my %known_filters = (); + if ($colData{STAGE}[$i] eq "Not Set") { + $colData{STAGE}[$i] = 'diff'; + } + if ($colData{OBSCODE}[$i] eq "Not Set") { + $colData{OBSCODE}[$i] = 566; + } +# Define filter and MJD from FPA_ID + if (($colData{FILTER}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) { + if (exists($known_filters{$colData{FPA_ID}[$i]})) { + $colData{FILTER}[$i] = $known_filters{$colData{FPA_ID}[$i]}; + } + else { + my $cmd = "$regtool -processedexp -dbname $dbname -exp_name $colData{FPA_ID}[$i]"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + # This is a problem, because I'm not sure how we handle a failure to read something. + # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, + # we can't send that response back. + die("Unable to perform $cmd error code: $error_code"); + } + foreach my $entry (split /\n/, (join "", @$full_buf)) { + $entry =~ s/^\s+//; + my @line = split /\s+/, $entry; + if (scalar(@line) != 3) { + next; + } + my ($key,$type,$value) = @line; +# print "$entry => $key / $type / $value \n"; + if ($key =~ /filter/i) { + $value =~ s/.00000//; + $colData{FILTER}[$i] = $value; + $known_filters{$colData{FPA_ID}[$i]} = $value; + } + } + } + } + if (($colData{'MJD-OBS'}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) { + # HACK! + my $mjd = $colData{FPA_ID}[$i]; + $mjd =~ s/o(....)g.*/$1/; + $mjd += 50000; + $colData{'MJD-OBS'}[$i] = $mjd; + } + if (($colData{FPA_ID}[$i] eq "Not Set")&&(($colData{'FILTER'}[$i] ne "Not Set")&& + ($colData{'MJD-OBS'}[$i] ne "Not Set"))) { + my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]); + my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1); + my $ra = $colData{'RA1_DEG'}[$i]; + my $dec = $colData{'DEC1_DEG'}[$i]; + my $filter = $colData{'FILTER'}[$i] . ".00000"; + my $cmd = "$camtool -processedexp -dbname $dbname -filter $filter "; + $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end "; + $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + # This is a problem, because I'm not sure how we handle a failure to read something. + # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, + # we can't send that response back. + die("Unable to perform $cmd error code: $error_code"); + } + foreach my $entry (split /\n/, (join "", @$full_buf)) { + $entry =~ s/^\s+//; + my @line = split /\s+/, $entry; + if (scalar(@line) != 3) { + next; + } + my ($key,$type,$value) = @line; +# print "$entry => $key / $type / $value \n"; + if ($key =~ /exp_name/i) { + $colData{FPA_ID}[$i] = $value; + } + } } } @@ -189,4 +254,5 @@ foreach my $colName (@unique_fields) { my %counter = (); + my $i = 0; foreach my $row (@{ $colData{$colName} }) { $counter{$row} = 1; @@ -232,5 +298,5 @@ my $msg; # Message to output Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); - die "CFITSIO error: $msg\n"; + die "CFITSIO error: $status => $msg\n"; } } @@ -252,4 +318,15 @@ } +# Stolen from PStamp/Job.pm. Thanks, Bill. +sub mjd_to_dateobs { + my $mjd = shift; + + my $ticks = ($mjd - 40587.0) * 86400; + + my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks); + + return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec; +} + Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/detectability_respond.pl =================================================================== --- /branches/eam_branches/ipp-20100621/pstamp/scripts/detectability_respond.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/pstamp/scripts/detectability_respond.pl (revision 28980) @@ -108,5 +108,5 @@ # Parse input request file using detect_query_read (as it's already written). # - my $dqr_command = "$detect_query_read --input $request_file"; + my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $dqr_command, verbose => $verbose); @@ -308,4 +308,6 @@ } close(WISDOM); + +# If there is anything that needs to be updated, create the update request list, and then exit the program. my $exit_code = 0; my $update_request_file = "${workdir}/update_request.dat"; @@ -316,4 +318,7 @@ $exit_code = 25; } + elsif ($fault != 0) { + $exit_code = 21; + } my $update_request = join ' ', @{ $update_request{$images}{$fault} }; print UPDATE_REQUEST "$update_request\n"; @@ -321,7 +326,7 @@ } close(UPDATE_REQUEST); -# if ($exit_code != 0) { -# exit($exit_code); -# } +if ($exit_code != 0) { + exit($exit_code); +} # This duplicates stuff returned by PSTAMP, but my thought is to convert that ---^ into a conditional, in which I read @@ -338,5 +343,4 @@ } } -#exit(10); # run ppCoord and psphotForced to calculate the required data. @@ -569,5 +573,5 @@ $columns = [ # matching rownum from detectability original request - { name => 'ROWNUM', type => 'V', writetype => TULONG }, + { name => 'ROWNUM', type => '20A', writetype => TSTRING }, # any errors that occurred during processing { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, @@ -715,6 +719,6 @@ # print STDERR "$writeType $i $numRows $colName $status @{ $colData{$colName} }\n"; $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status ); +# print "$writeType $i $numRows $colName $status\n"; check_fitsio( $status ); - } $outFits->close_file( $status ); @@ -730,4 +734,5 @@ my $msg; # Message to output Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + carp("CFITSIO error: $status => $msg"); die "CFITSIO error: $msg\n"; } Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/dqueryparse.pl =================================================================== --- /branches/eam_branches/ipp-20100621/pstamp/scripts/dqueryparse.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/pstamp/scripts/dqueryparse.pl (revision 28980) @@ -158,5 +158,5 @@ } } -else { +elsif ($fault == $PSTAMP_NOT_AVAILABLE) { # Failed to run correctly, which means that we need to queue a job and flag data for updating. # Get the dependency id for the data we're requesting be updated. @@ -191,4 +191,7 @@ close(UPDATE_REQUEST); } +else { + my_die ("Parse fault!!", $PS_EXIT_UNKNOWN_ERROR); +} # This does not set the request state to stop. That will happen with the request_finish.pl script, Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_checkdependent.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_checkdependent.pl (revision 28980) @@ -147,5 +147,5 @@ # been updated we need to go and queue it. - if ($it->{fault}) { + if (($it->{state} ne 'cleaned') and $it->{fault}) { my_die("Component faulted on update dep_id: $dep_id", $PS_EXIT_SYS_ERROR); Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_job_run.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_job_run.pl (revision 28980) @@ -93,6 +93,14 @@ $argString .= " -file $params->{image}"; - $argString .= " -mask $params->{mask}"; - my @file_list = ($params->{image}, $params->{mask}); + my @file_list = ($params->{image}); + + my $nan_masked = 1; + if (!$params->{magicked}) { + $nan_masked = 0; + } + if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) { + $argString .= " -mask $params->{mask}"; + push @file_list, $params->{mask}; + } if ($options & $PSTAMP_SELECT_VARIANCE) { $argString .= " -variance $params->{weight}"; @@ -351,5 +359,5 @@ foreach my $f (@_) { if (!$ipprc->file_exists($f)) { - my_die( "file $f does not exist:", $job_id, $PS_EXIT_SYS_ERROR); + my_die( "file $f does not exist:", $job_id, $PSTAMP_GONE, 'stop'); } } @@ -361,6 +369,8 @@ my $job_id = shift; # job identifier my $exit_code = shift; # Exit code to add + my $job_state = shift; # new pstampJob.state $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code; + $job_state = 'run' unless $job_state; carp($msg); @@ -370,5 +380,5 @@ $command .= " -set_fault $exit_code"; # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided - $command .= " -set_state run"; + $command .= " -set_state $job_state"; $command .= " -dbname $dbname" if defined $dbname; $command .= " -dbserver $dbserver" if defined $dbserver; Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/pstampparse.pl =================================================================== --- /branches/eam_branches/ipp-20100621/pstamp/scripts/pstampparse.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/pstamp/scripts/pstampparse.pl (revision 28980) @@ -386,9 +386,10 @@ $need_magic = $proj_hash->{need_magic}; - # Temporary hack so that MOPS can get at non-magicked data + # Temporary hack so that IFA can get at non-magicked data my $allow_mops_unmagicked = 1; if ($allow_mops_unmagicked) { if ($product and (($product eq "mops-pstamp-results") or - ($product eq "mops-pstamp-results2"))) { + ($product eq "mops-pstamp-results2") or + ($product eq "ifa-pstamp-results"))) { $need_magic = 0; } Index: /branches/eam_branches/ipp-20100621/pstamp/src/ppstampMakeStamp.c =================================================================== --- /branches/eam_branches/ipp-20100621/pstamp/src/ppstampMakeStamp.c (revision 28979) +++ /branches/eam_branches/ipp-20100621/pstamp/src/ppstampMakeStamp.c (revision 28980) @@ -229,5 +229,5 @@ static int makeStamp(pmConfig *config, ppstampOptions *options, pmFPAfile *input, - pmChip *inChip, pmFPAview *view, pmAstromObj *center) + pmChip *inChip, pmFPAview *view, pmAstromObj *center, pmFPAfile *astrom) { int status = false; @@ -266,4 +266,9 @@ srcFile = input; *srcView = *view; + } + + if (astrom->camera != srcFile->camera) { + psError(PS_ERR_UNKNOWN, true, "Input camera and astrometry camera do not match"); + return PS_EXIT_CONFIG_ERROR; } @@ -583,7 +588,4 @@ if (!astrom) { astrom = input; - } else if (astrom->camera != input->camera) { - psError(PS_ERR_UNKNOWN, true, "Input camera and astrometry camera do not match"); - return PS_EXIT_CONFIG_ERROR; } @@ -636,5 +638,5 @@ case PPSTAMP_ON: case PPSTAMP_PARTIALLY_ON: - returnval = makeStamp(config, options, input, chip, view, center); + returnval = makeStamp(config, options, input, chip, view, center, astrom); allDone = true; foundOverlap = true; Index: /branches/eam_branches/ipp-20100621/tools/czarplot.pl =================================================================== --- /branches/eam_branches/ipp-20100621/tools/czarplot.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/czarplot.pl (revision 28980) @@ -21,4 +21,5 @@ my $histogram = undef; my $timeSeries = undef; +my $nebulous = undef; my $savingToFile = undef; @@ -32,4 +33,5 @@ "output|o=s" => \$path, "histogram|h" => \$histogram, + "nebulous|n" => \$nebulous, "timeseries|t" => \$timeSeries, "verbose|v" => \$verbose, @@ -41,6 +43,8 @@ if (!$timeSeries) { print "* OPTIONAL: plot timeseries -t (default=on)\n";} +if (!$nebulous) { + print "* OPTIONAL: plot nebulous disk space -n (default=off)\n";} if (!$label) { - print "* OPTIONAL: choose a label -l (default='all_labels')\n";} + print "* OPTIONAL: choose a label -l (default='all_stdscience_labels')\n";} if (!$stage) { print "* OPTIONAL: choose a stage -s (default=none)\n";} @@ -57,6 +61,6 @@ # default values -if (!$label) {$label = "all_labels";} -if (!$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 1;} +if (!$label) {$label = "all_stdscience_labels";} +if (!$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 0;} if (!$verbose) {$verbose = 0;} if (!$save_temps) {$save_temps = 0;} @@ -67,5 +71,5 @@ $czarDb->setDateFormat("%Y%m%d-%H%i%s"); -my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png" : "X11", $path, $save_temps); +my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", $path, $save_temps); my @allStages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); @@ -78,5 +82,9 @@ } -if ($timeSeries) {$czarplot->createTimeSeries($label, $stage, $begin, $end);} +my $diskUsage = 1; # TODO + +if (!$nebulous && $timeSeries) {$czarplot->createTimeSeries($label, $stage, $begin, $end);} if ($histogram) {$czarplot->createHistogram($label, $begin, $end);} +if ($nebulous && $timeSeries) {$czarplot->plotStorageTimeSeries($begin, $end);} +elsif ($nebulous) {$czarplot->plotDiskUsageHistogram();} Index: /branches/eam_branches/ipp-20100621/tools/czartool.pl =================================================================== --- /branches/eam_branches/ipp-20100621/tools/czartool.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/czartool.pl (revision 28980) @@ -64,6 +64,6 @@ if ($key eq "s") {checkServers();} - elsif ($key eq "l") {checkAllLabels("new");} - elsif ($key eq "u") {serverStatus("stdscience");} + elsif ($key eq "l") {checkAllLabels("stdscience");} + elsif ($key eq "u") {$pantasks->getServerCurrentStatus("stdscience");} elsif ($key =~ m/[0-9]/) {my $key2=getc; checkOneLabel($stdscienceLabels[($key.$key2)-1]);} printInstructions(); @@ -99,22 +99,4 @@ return \@diff; -} - -########################################################################### -# -# Prints the status of a given pantasks server -# -########################################################################### -sub serverStatus { - my ($server) = @_; - - my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; - my $line; - foreach $line (@cmdOut) { - - chomp($line); - - print "$line\n"; - } } @@ -263,7 +245,7 @@ my $new = $gpc1Db->countExposures($label, $stage, $state); - if ($state ne "new") {return $new;} - - my $faults = $gpc1Db->countFaults($label, $stage); + #if ($state ne "new") {return $new;} + + my $faults = $gpc1Db->countFaults($label, $stage, $state); if ($faults < 1) {return $new;} Index: /branches/eam_branches/ipp-20100621/tools/czartool/CzarDb.pm =================================================================== --- /branches/eam_branches/ipp-20100621/tools/czartool/CzarDb.pm (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/czartool/CzarDb.pm (revision 28980) @@ -16,5 +16,5 @@ # Call the constructor of the parent class, Person. - my $self = $class->SUPER::new( $_[1], $_[2], $_[3], $_[4], $_[5], $_[6]); + my $self = $class->SUPER::new($_[1], $_[2], $_[3], $_[4], $_[5], $_[6]); bless $self, $class; @@ -66,4 +66,39 @@ $query->execute; +} + +########################################################################### +# +# Gets revert status for this stage +# +########################################################################### +sub getRevertStatus { + my ($self, $stage, $reverting) = @_; + + my $query = $self->{_db}->prepare(<execute; + return scalar $query->fetchrow_array(); +} + +########################################################################### +# +# Updates revert status for this stage +# +########################################################################### +sub updateRevertStatus { + my ($self, $stage, $reverting) = @_; + + my $query = $self->{_db}->prepare(<execute; } @@ -117,4 +152,46 @@ } } + +########################################################################### +# +# Updates hosts table +# +########################################################################### +sub updateHost { + my ($self, $host, $total, $available, $used, $readable, $writable) = @_; + + my $query = $self->{_db}->prepare(<execute; + $query = $self->{_db}->prepare(<execute; + +} + +########################################################################### +# +# Inserts new cluster space info +# +########################################################################### +sub insertNewClusterSpace { + my ($self, $total, $available, $used, $hostsOver98) = @_; + + my $query = $self->{_db}->prepare(<execute; +} + ########################################################################### # @@ -123,11 +200,11 @@ ########################################################################### sub insertNewTimeData { - my ($self, $stage, $label, $pending, $processed, $faults, $reverting) = @_; + my ($self, $stage, $label, $pending, $processed, $faults) = @_; my $query = $self->{_db}->prepare(<execute) {return undef;} - my $minProcessed; + my $minProcessed = undef; ($minProcessed, ${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array(); - if (!$minProcessed) {return undef;} + if (!defined $minProcessed) {return undef;} $query = $self->{_db}->prepare(<$dataFile"); + + my $query = $self->{_db}->prepare(<execute; + + my $underLimit; + + # loop round results + while (my @row = $query->fetchrow_array()) { + + my ($host, $used, $available, $writable, $readable) = @row; + + if (($used/($used+$available))*100 > $limit) {$underLimit = 0;} + else {$underLimit = 1;} + + # Col 1: host + print GNUDAT "$host"; + + # Col 2: available, readable used space under limit + if ($readable && $underLimit) {print GNUDAT " $used";} + else {print GNUDAT " 0";} + + # Col 3: available, readable used space OVER limit + if ($readable && !$underLimit) {print GNUDAT " $used";} + else {print GNUDAT " 0";} + + # Col 4: NOT available used space + if (!$readable) {print GNUDAT " $used";} + else {print GNUDAT " 0";} + + # Col 5: available writable space + if($writable) {print GNUDAT " $available";} + else {print GNUDAT " 0";} + + # Col 6: NOT available writable space + if(!$writable) {print GNUDAT " $available";} + else {print GNUDAT " 0";} + + print GNUDAT "\n"; + } + close(GNUDAT); + + return $dataFile; +} + +########################################################################### +# +# Gets total used cluster space as a percentage +# +############################################################################ +sub getTotalClusterStorageAsPercentage { + my ($self) = @_; + + my $query = $self->{_db}->prepare(<execute; + return scalar $query->fetchrow_array(); +} + +########################################################################### +# +# Gets time series data for cluster storage and saves it to file +# +########################################################################### +sub createStorageTimeSeriesData { + my ($self, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_; + + my $dataFile = "/tmp/czarplot_gnuplot_storage_timeseries.dat"; + open (GNUDAT, ">$dataFile"); + + my $query = $self->{_db}->prepare(<{_dateFormat}'), + DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'), + TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp))) + FROM cluster_space + WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'; +SQL + + + if (!$query->execute) {return undef;} + + (${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array(); + + + $query = $self->{_db}->prepare(<{_dateFormat}'), available + FROM cluster_space + WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime' + ORDER BY timestamp; +SQL + + $query->execute; + + # loop round results + while (my @row = $query->fetchrow_array()) { + + my ($timestamp, $available) = @row; + print GNUDAT "$timestamp $available\n"; + } + close(GNUDAT); + + return $dataFile; +} +########################################################################### +# # Determines how much has been processed in the provided interval of time # (format: 1 HOUR, 1 MINUTE 1 DAY etc) @@ -301,5 +500,5 @@ my $currentRevision = -1; - my $latestRevision = 7; + my $latestRevision = 10; while ($currentRevision != $latestRevision) { @@ -315,5 +514,44 @@ elsif ($currentRevision == 5) {$self->createRevision_6();} elsif ($currentRevision == 6) {$self->createRevision_7();} - } + elsif ($currentRevision == 7) {$self->createRevision_8();} + elsif ($currentRevision == 8) {$self->createRevision_9();} + elsif ($currentRevision == 9) {$self->createRevision_10();} + } +} + +####################################################################################### +# +# Sets current revision of ippToPsps database +# +####################################################################################### +sub setRevision { + my ($self, $revision) = @_; + + my $query = $self->{_db}->prepare(<execute; +} + +####################################################################################### +# +# Gets current revision of ippToPsps database +# +####################################################################################### +sub getRevision { + my ($self) = @_; + + if (!$self->SUPER::doesTableExist("revision")) {return 0;} + + my $query = $self->{_db}->prepare(<execute; + my @row = $query->fetchrow_array(); + + return $row[0]; } @@ -500,36 +738,105 @@ ####################################################################################### # -# Sets current revision of ippToPsps database -# -####################################################################################### -sub setRevision { - my ($self, $revision) = @_; - - my $query = $self->{_db}->prepare(<execute; -} - -####################################################################################### -# -# Gets current revision of ippToPsps database -# -####################################################################################### -sub getRevision { - my ($self) = @_; - - if (!$self->SUPER::doesTableExist("revision")) {return 0;} - - my $query = $self->{_db}->prepare(<execute; - my @row = $query->fetchrow_array(); - - return $row[0]; +# Create revision 8 of the database +# +####################################################################################### +sub createRevision_8 { + my ($self) = @_; + + print "* Creating revision 8 of '$self->{_dbName}'\n"; + + # drop reverting column from all stages tables + my $stage = undef; + foreach $stage (@stages) { + + my $query = $self->{_db}->prepare(<execute; + } + + # create new 'revert' table + my $query = $self->{_db}->prepare(<execute; + + # insert stages into revert table + foreach $stage (@stages) { + my $query = $self->{_db}->prepare(<execute; + } + + $self->setRevision(8); +} + +####################################################################################### +# +# Create revision 9 of the database +# +####################################################################################### +sub createRevision_9 { + my ($self) = @_; + + print "* Creating revision 9 of '$self->{_dbName}'\n"; + + my $query = $self->{_db}->prepare(<execute; + $query = $self->{_db}->prepare(<execute; + + $query = $self->{_db}->prepare(<execute; + + $self->setRevision(9); +} + +####################################################################################### +# +# Create revision 10 of the database +# +####################################################################################### +sub createRevision_10 { + my ($self) = @_; + + print "* Creating revision 10 of '$self->{_dbName}'\n"; + + my $query = $self->{_db}->prepare(<execute; + + $self->setRevision(10); } Index: /branches/eam_branches/ipp-20100621/tools/czartool/Czarplot.pm =================================================================== --- /branches/eam_branches/ipp-20100621/tools/czartool/Czarplot.pm (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/czartool/Czarplot.pm (revision 28980) @@ -154,10 +154,57 @@ my ($self, $minY, $maxY) = @_; - ${$maxY} = ${$maxY} + (${$maxY}/10); - ${$minY} = ${$minY} - (${$minY}/10); + my $tmp; + if (${$maxY} < ${$minY}) { + + $tmp = ${$maxY}; + ${$maxY} = ${$minY}; + ${$minY} = $tmp; + } + + my $border = (${$maxY} - ${$minY})/10; + + ${$maxY} += $border; + ${$minY} -= $border ; if (${$maxY} == 0) {${$maxY} = 1;} if (${$minY} == 0) {${$minY} = -1;} } + +########################################################################### +# +# Figures out suitable spacing for time tics on time-series pliots +# +########################################################################### +sub getTimeSpacing { + my ($self, $timeDiff, $divX, $timeFormat) = @_; + + # if less than a couple of hour's data plotted, show 30 mins tics + if ($timeDiff < 7200) { + ${$timeFormat} = "%H:%M"; + ${$divX} = 1800; + } + # if less than half a day's data plotted, show hourly tics + elsif ($timeDiff < 43200) { + ${$timeFormat} = "%H:%M"; + ${$divX} = 3600; + } + # if less than one day's data plotted, show 2 hourly tics + elsif ($timeDiff < 86400) { + ${$timeFormat} = "%H:%M"; + ${$divX} = 7200; + } + # if less than 1 week's data is data plotted, show daily tics + elsif ($timeDiff < 604800) { + + ${$timeFormat} = "%m/%d"; + ${$divX} = 86400; + } + else { + + ${$timeFormat} = "%m/%d"; + ${$divX} = 172800; + } +} + ########################################################################### # @@ -171,30 +218,16 @@ my $timeFormat = undef; - - my $divX = $timeDiff/4; - - # if less than a couple of hour's data plotted, show 30 mins tics - if ($timeDiff < 7200) { - $timeFormat = "%H:%M"; - $divX = 1800; - } - # if less than half a day's data plotted, show hourly tics - elsif ($timeDiff < 43200) { - $timeFormat = "%H:%M"; - $divX = 3600; - } - # if less than one day's data plotted, show 2 hourly tics - elsif ($timeDiff < 86400) { - $timeFormat = "%H:%M"; - $divX = 7200; - } - # if more than one day's data plotted, show daily tics - else { - - $timeFormat = "%m/%d %H:%M"; - $divX = 86400; - } + my $divX = undef; + + $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat); my $numOfPlots = keys %$gnuplotFiles; + my $title = undef; + + # sort out plot title + if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title = "'".$stage."'";}} + else {$title = "'All stages'"} + + $title .= " for '$label', '$fromTime' to '$toTime'"; open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; @@ -205,8 +238,9 @@ "set term $self->{_outputFormat};" . "set output \"$outputFile\";" . - "set title \"All stages for '$label' since $fromTime\";" . + "set title \"$title\";" . "set key left top;" . "set xdata time;" . "set timefmt \"$self->{_dateFormat}\";" . + # "set yrange [\"$minY\":\"$maxY\"];" . "set xrange [\"$minX\":\"$maxX\"];" . "set format x \"$timeFormat\";" . @@ -224,7 +258,7 @@ if ($numOfPlots == 1) { - print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"processed\" with lines lt 2,"; - print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"pending\" with lines lt 3,"; - print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"faults\" with lines lt 1"; + print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2,"; + print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4,"; + print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 7"; } # when plotting multiple stages, show only processed @@ -241,9 +275,22 @@ ########################################################################### # -# Plots a histogram of processed stuff -# -########################################################################### -sub plotHistogram { - my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime) = @_; +# Plots a time-series of cluster storage +# +########################################################################### +sub plotStorageTimeSeries { + my ($self, $fromTime, $toTime) = @_; + + my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; + my $outputFile = "$prefix/czarplot_cluster.png"; + my ($minX, $maxX, $minY, $maxY, $timeDiff); + + my $gnuplotFile = $self->{_czarDb}->createStorageTimeSeriesData($fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff); + + $self->sortOutMaxMinY(\$minY, \$maxY); + + my $timeFormat = undef; + my $divX = undef; + + $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat); open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; @@ -254,5 +301,39 @@ "set term $self->{_outputFormat};" . "set output \"$outputFile\";" . - "set title \"Processing status for '$label' since $fromTime\";" . + "set title \"Total available cluster space over time\";" . + "set key left top;" . + "set xdata time;" . + "set timefmt \"$self->{_dateFormat}\";" . + "set xrange [\"$minX\":\"$maxX\"];" . + "set yrange [\"$minY\":\"$maxY\"];" . + "set format x \"$timeFormat\";" . + "set xtics \"$minX\", $divX, \"$maxX\";" . + "set grid xtics;" . + "set xlabel \"Time\";" . + "set ylabel \"Available (TB)\";" . + "plot " . +# "'$gnuplotFile' using 1:2 title \"Total\" with lines lt 1," . + "'$gnuplotFile' using 1:2 title \"Available\" with lines lt 2\n"; + + print GP "\n"; + close GP; +} + +########################################################################### +# +# Plots a histogram of processed stuff +# +########################################################################### +sub plotHistogram { + my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime) = @_; + + open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; + use FileHandle; + GP->autoflush(1); + + print GP + "set term $self->{_outputFormat};" . + "set output \"$outputFile\";" . + "set title \"'$label', '$fromTime' to '$toTime'\";" . "set grid;" . "set boxwidth;" . @@ -262,8 +343,51 @@ "set bmargin 5;" . "set ylabel \"Exposures\";" . - "plot '$inputFile' using 2:xtic(1) title \"Processed\" lt 2, '' using 3 title \"Pending\" lt 3, '' using 4 title \"Faults\" lt 1\n"; + "plot '$inputFile' using 2:xtic(1) title \"Processed\" lt 2, '' using 3 title \"Pending\" lt 4, '' using 4 title \"Faults\" lt 7;" . + "\n"; close GP; } +########################################################################### +# +# Plots disk usage across cluster as a stacked histogram +# +########################################################################### +sub plotDiskUsageHistogram { + my ($self) = @_; + + my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this + my $outputFile = "$prefix/czarplot_hosts_space.png"; + my $limit = 97.0; + my $inputFile = $self->{_czarDb}->createHostsData($limit); + + my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage(); + + my $totalPercent = sprintf("%.1f%%", $totalUsed); + + open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; + use FileHandle; + GP->autoflush(1); + + print GP + "set term $self->{_outputFormat};" . + "set output \"$outputFile\";" . + "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" . + "set style fill solid 1.00 border -1;" . + "set style histogram rowstacked;" . + "set style data histograms;" . + "set ylabel \"Space (TB)\";" . + "set xtic rotate by -90 scale 0;" . + "plot '$inputFile' " . + "using 2:xtic(1) t \"Used\" lt 7," . + "'' using 3 t \"Over $limit% used\" lt 1," . + "'' using 4 t \"Unavailable\" fs solid 0.50 lt -1 ," . + "'' using 5 t \"Free\" lt 2," . + "'' using 6 notitle fs solid 0.50 lt -1" . + ";" . + + "\n"; + + close GP; +} 1; Index: /branches/eam_branches/ipp-20100621/tools/czartool/Gpc1Db.pm =================================================================== --- /branches/eam_branches/ipp-20100621/tools/czartool/Gpc1Db.pm (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/czartool/Gpc1Db.pm (revision 28980) @@ -8,4 +8,32 @@ use czartool::MySQLDb; our @ISA = qw(czartool::MySQLDb); # inherits from MySQLDb + +########################################################################### +# +# Returns the right join table and ID to join on for for a given stage TODO make private +# +########################################################################### +sub getJoinTableAndId { + my ($self, $stage, $joinTable, $id) = @_; + + ${$id} = $stage."_id"; + + if ($stage eq "chip") {${$joinTable}="chipProcessedImfile";} + elsif ($stage eq "cam") {${$joinTable}="camProcessedExp";} + elsif ($stage eq "fake") {${$joinTable}="fakeProcessedImfile";} + elsif ($stage eq "warp") {${$joinTable}="warpSkyfile";} + elsif ($stage eq "stack") {${$joinTable}="stackSumSkyfile";} + elsif ($stage eq "diff") {${$joinTable}="diffSkyfile";} + elsif ($stage eq "magic") {${$joinTable}="magicNodeResult";} + elsif ($stage eq "magicDS") {${$id} = "magic_ds_id"; ${$joinTable}="magicDSFile";} + elsif ($stage eq "dist") {${$joinTable}="distComponent";} + else { + + print "* ERROR: could nod find joinTable and ID for '$stage' stage\n"; + return 0; + } + + return 1; +} ########################################################################### @@ -30,5 +58,4 @@ } - ########################################################################### # @@ -37,21 +64,11 @@ ########################################################################### sub countFaults { - my ($self, $label, $stage) = @_; + my ($self, $label, $stage, $state) = @_; - - my $table = getTableForStage($self, $stage); - my $joinTable; - my $id = $stage."_id"; - - if ($stage eq "chip") {$joinTable="chipProcessedImfile";} - elsif ($stage eq "cam") {$joinTable="camProcessedExp";} - elsif ($stage eq "fake") {$joinTable="fakeProcessedImfile";} - elsif ($stage eq "warp") {$joinTable="warpSkyfile";} - elsif ($stage eq "stack") {$joinTable="stackSumSkyfile";} - elsif ($stage eq "diff") {$joinTable="diffSkyfile";} - elsif ($stage eq "magic") {$joinTable="magicNodeResult";} - elsif ($stage eq "magicDS") {$id = "magic_ds_id"; $joinTable="magicDSFile";} - elsif ($stage eq "dist") {$joinTable="distComponent";} - else {return -1;} + my $table = undef; + my $joinTable = undef; + my $id = undef; + $table = getTableForStage($self, $stage); + if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;} my $faultCol = $joinTable.".fault"; @@ -64,5 +81,5 @@ WHERE label LIKE '$label' AND $faultCol != 0 - AND $stateCol = 'new' + AND $stateCol = '$state' SQL @@ -85,5 +102,5 @@ SQL - $query->execute; + $query->execute; my $priority = scalar $query->fetchrow_array(); if (!$priority) {return 50000;} # assume labels not given priority in gpc1 Db have highest priority @@ -100,15 +117,47 @@ my $table = getTableForStage($self, $stage); - if ($state eq "fault") {return countFaults($stage);} + my $query = undef; + + if ($state eq "update") { + + if ($stage eq "dist") {return 0;} + my $joinTable = undef; + my $id = undef; + if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;} + + if ($stage eq "chip" || $stage eq "fake" || $stage eq "warp" || $stage eq "diff" || $stage eq "magicDS") { + $query = $self->{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<execute; + $query->execute; return scalar $query->fetchrow_array(); } Index: /branches/eam_branches/ipp-20100621/tools/czartool/Nebulous.pm =================================================================== --- /branches/eam_branches/ipp-20100621/tools/czartool/Nebulous.pm (revision 28980) +++ /branches/eam_branches/ipp-20100621/tools/czartool/Nebulous.pm (revision 28980) @@ -0,0 +1,161 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; + +package czartool::Nebulous; + +########################################################################### +# +# Constructor TODO put in check that we can run nebulous +# +########################################################################### +sub new { + my $class = shift; + my $self = {_czarDb => shift}; + + $self->{_totalAvail} = undef; + $self->{_totalUsed} = undef; + $self->{_totalUsedAsPercent} = undef; + $self->{_totalCluster} = undef; + $self->{_totalHosts} = undef; + $self->{_hostsOverNinetyEightPC} = undef; + + bless $self, $class; + return $self; +} + +########################################################################### +# +# Converts Kb to Tb +# +########################################################################### +sub convertKbToTb { + my ($self, $Kb) = @_; + + return $Kb/1073741824; +} + +########################################################################### +# +# Returns total cluster usage as percentage +# +########################################################################### +sub getTotalUsedAsPercentage { + my ($self) = @_; + + return $self->{_totalUsedAsPercent}; +} + +########################################################################### +# +# Returns total cluster usage in T +# +########################################################################### +sub getTotalUsedInTb { + my ($self) = @_; + + return $self->convertKbToTb($self->{_totalUsed}); +} + +########################################################################### +# +# Returns total cluster space in T +# +########################################################################### +sub getTotalClusterInTb { + my ($self) = @_; + + return $self->convertKbToTb($self->{_totalCluster}); +} + +########################################################################### +# +# Returns total cluster available in T +# +########################################################################### +sub getTotalAvailInTb { + my ($self) = @_; + + return $self->convertKbToTb($self->{_totalAvail}); +} + +########################################################################### +# +# Runs neb-df and puts output in gnuplot ready format +# +########################################################################### +sub updateClusterSpaceInfo { + my ($self) = @_; + + my @cmdOut = `neb-df`; + my $line; + $self->{_totalHosts} = 0; + $self->{_hostsOverNinetyEightPC} = 0; + my $readable; + my $writable; + foreach $line (@cmdOut) { + + chomp($line); + if ($line =~ m/.*Filesystem.*/i) {next;} + elsif ($line =~ m/.*summmary.*/i) {next;} + elsif ($line =~ m/.*total.*/i) {next;} + elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*allocated.*/i) { + + $self->{_totalUsed} = $self->convertKbToTb($1); + $self->{_totalAvail} = $self->convertKbToTb($2); + $self->{_totalCluster} = $self->convertKbToTb($1 + $2); + $self->{_totalUsedAsPercent} = ($self->{_totalUsed}/$self->{_totalCluster})*100; + + next; + } + elsif ($line =~ m/(ipp[0-9]+)\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) { + + $self->{_totalHosts}++; + if ($4 >= 98) {$self->{_hostsOverNinetyEightPC}++;} + + $self->getReadableWritableStatus($1, \$readable, \$writable); + + $self->{_czarDb}->updateHost($1, + $self->convertKbToTb($2+$3), + $self->convertKbToTb($3), + $self->convertKbToTb($2), + $readable, $writable); + } + } + + $self->{_czarDb}->insertNewClusterSpace( + $self->{_totalCluster}, + $self->{_totalAvail}, + $self->{_totalUsed}, + $self->{_hostsOverNinetyEightPC}); +} + +########################################################################### +# +# Runs neb-host and readable/writable status for this host +# +########################################################################### +sub getReadableWritableStatus { + my ($self, $host, $readable, $writable) = @_; + + ${$readable} = 0; + ${$writable} = 0; + + my @cmdOut = `neb-host`; + my $line; + foreach $line (@cmdOut) { + + chomp($line); + if ($line =~ m/$host\s+[0-9.A-Za-z]+\s+([01]+)\s+([01]+)\s+([01]+)\s+.*/i) { + + # if mounted... + if ($1) { + + ${$readable} = $3; + ${$writable} = $2; + } + } + } +} +1; Index: /branches/eam_branches/ipp-20100621/tools/czartool/Pantasks.pm =================================================================== --- /branches/eam_branches/ipp-20100621/tools/czartool/Pantasks.pm (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/czartool/Pantasks.pm (revision 28980) @@ -5,5 +5,4 @@ package czartool::Pantasks; - my @servers = ( @@ -21,4 +20,16 @@ ); +########################################################################### +# +# Constructor TODO put in check that we can run pantasks_client +# +########################################################################### +sub new { + my $class = shift; + my $self = {}; + + bless $self, $class; + return $self; +} ########################################################################### @@ -35,17 +46,4 @@ ########################################################################### # -# Constructor -# -########################################################################### -sub new { - my $class = shift; - my $self = {}; - - bless $self, $class; - return $self; -} - -########################################################################### -# # Returns the correct server for this processing stage # @@ -58,4 +56,22 @@ } +########################################################################### +# +# Checks for a meaningful response from pantasks client +# +########################################################################### +sub outputOk { + my ($self, $output) = @_; + + if ( + $output =~ m/.*show\.labels.*/i || + $output =~ m/.*Connection reset by peer.*/i || + $output =~ m/.*server is busy.*/i ) { + + print "* WARNING: pantasks returned '$output'\n"; + return 0; + } + return 1; +} ########################################################################### @@ -75,18 +91,6 @@ chomp($line); - if ($line =~ m/pantasks:\s+(.*)/) { - - # HACK quit if we get 'show.labels: Command not found.' - if ($1 =~ m/.*show\.labels.*/i) {return \@labels;}; - # HACK quit if we get 'Connection reset by peer' - if ($1 =~ m/.*Connection reset by peer.*/i) {return \@labels;}; - # HACK quit if we get 'server is busy' message - if ($1 =~ m/.*server is busy.*/i) {return \@labels;}; - # HACK to get around 'dummy' label in stdscience - if ($1 !~ m/.*dummy.*/) {push(@labels, $1);}; - $passedHeader=1; - next; - } - + if (!$self->outputOk($line)) {return \@labels;} + if ($line =~ m/pantasks:\s+/) {$passedHeader=1; next;} if ($passedHeader) {push(@labels, $line);} } @@ -113,5 +117,4 @@ if ($line =~ m/Scheduler is running/) {${$running} = 1; ${$alive}=1;last;} if ($line =~ m/Task Status/) {last;} - } } @@ -125,15 +128,38 @@ my ($self, $server) = @_; + my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; + my $line; + my $passedHeader=0; + print "\n"; + foreach $line (@cmdOut) { + + if ($line =~ m/.*Task Status.*/) {$passedHeader=1;next;} + if ($passedHeader){print "$line";} + } +} + +########################################################################### +# +# Gets revert status for this stage +# +########################################################################### +sub getRevertStatus { + my ($self, $stage, $reverting) = @_; + + my $server = $self->getServerForThisStage($stage); + + if ($stage eq "cam") {$stage = "camera";} my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`; my $line; - my $passedHeader=0; foreach $line (@cmdOut) { chomp($line); - if ($line =~ m/.*Task Status.*/) {$passedHeader=1;next;} - if ($passedHeader){print "$line
";} + if ($line =~ m/\s+([+-])[+-]\s+$stage\.revert.*/) { + + if ($1 =~ m/-/) {${$reverting} = 0;} + elsif ($1 =~ m/\+/) {${$reverting}=1;} + } } } - - +1; Index: /branches/eam_branches/ipp-20100621/tools/difflog_grep.pl =================================================================== --- /branches/eam_branches/ipp-20100621/tools/difflog_grep.pl (revision 28980) +++ /branches/eam_branches/ipp-20100621/tools/difflog_grep.pl (revision 28980) @@ -0,0 +1,216 @@ +#!/usr/bin/env perl + +# +# find diff log files through difftool, and grep them +# + +use warnings; +use strict; + +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $missing_tools; +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $neblocate = can_run('neb-locate') or (warn "Can't find neb-locate" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + + + +my ($db_host, $dbname, $db_user, $db_pw, $dateobs_begin, $dateobs_end, $label, $filter, $data_group, $dist_group, $ra, $decl, $radius, $limit, $pstamp_order, $magicked, $not_destreaked, $destreaked, $grep, $diffandsky, $uselogname, $ippMonitor); # Database details +$db_host = 'ippdb01'; +$db_pw = 'ipp'; +$db_user = 'ipp'; +GetOptions( + ## 'dbhost=s' => \$db_host, # Database host name + 'dbname=s' => \$dbname, # Database name + ## 'dbuser=s' => \$db_user, # Database user + ## 'dbpass=s' => \$db_pw, # Database p/w + 'dateobs_begin=s' => \$dateobs_begin, # dateobs_begin + 'dateobs_end=s' => \$dateobs_end, # dateobs_end + 'label=s' => \$label, # label + 'filter=s' => \$filter, # filter + 'data_group=s' => \$data_group, # data_group + 'dist_group=s' => \$dist_group, # dist_group + 'grep=s' => \$grep, #thing to grep for + 'ra=s' => \$ra, # ra + 'decl=s' => \$decl, # ra + 'radius=s' => \$radius, # ra + 'limit' => \$limit, + 'pstamp_order' => \$pstamp_order, + 'magicked' => \$magicked, + 'not_destreaked' => \$not_destreaked, + 'destreaked' => \$destreaked, + 'diffandsky' => \$diffandsky, + 'ippMonitor' => \$ippMonitor, + # No magicked inputs? + ) or die "Unable to parse arguments.\n"; +die "Unknown option: @ARGV\n" if @ARGV; +die "difflog_grep.pl uses difftool and grep to look for 'something' in many diffs logfiles.\n\nRequired options: --dbhost --dbname --dbuser --dbpass\n\nOptional options: --dateobs_begin, --dateobs_end, --label, --filter, --data_group, --dist_group, --ra, --decl, --radius, --limit, --pstamp_order, --magicked, --not_destreaked, --destreaked will select diff_ids from difftool to investigate\n\n--grep - if grep is not used will just list the log files \n--diffandsky will print the diff_id and sky_id instead of the logfilename\n--ippMonitor will add ippMonitor like output (bg, nsources, etc)\n" + + unless defined $db_host + and defined $dbname + and defined $db_user + and defined $db_pw; + + + +my $mdcParser = PS::IPP::Metadata::Config->new; + + + my $fpaCommand1 = "$difftool -listrun"; + $fpaCommand1 .= " -dateobs_begin $dateobs_begin" if defined $dateobs_begin; + $fpaCommand1 .= " -dateobs_end $dateobs_end" if defined $dateobs_end; + $fpaCommand1 .= " -label $label" if defined $label; + $fpaCommand1 .= " -filter $filter" if defined $filter; + $fpaCommand1 .= " -data_group $data_group" if defined $data_group; + $fpaCommand1 .= " -dist_group $dist_group" if defined $dist_group; + $fpaCommand1 .= " -ra $ra" if defined $ra; + $fpaCommand1 .= " -decl $decl" if defined $decl; + $fpaCommand1 .= " -radius $radius" if defined $radius; + $fpaCommand1 .= " -limit " if defined $limit; + $fpaCommand1 .= " -pstamp_order " if defined $pstamp_order; + $fpaCommand1 .= " -magicked " if defined $magicked; + $fpaCommand1 .= " -not_destreaked " if defined $not_destreaked; + $fpaCommand1 .= " -destreaked " if defined $destreaked; + $fpaCommand1 .= " -dbname $dbname" if defined $dbname; + + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to list minidvodb database: $error_code\n"); + exit($error_code); + } + +if (defined(@$stdout_buf)){ + + if (defined($diffandsky)) { + print "diff_id skycell "; + } else {print "logfile ";} + + if (defined($ippMonitor)) { + print "bg bg_stdev dtime_diff dtime_match dtime_phot dtime_script stamps_num stamps_mean stamps_rms sources"; + } + + + print "\n"; + my $metadata2 = $mdcParser->parse(join "", @$stdout_buf) or + die "Unable to parse metadata config"; + + my $components2 = parse_md_list($metadata2) or + die "Unable to parse metadata list"; + my $comp2 = $$components2[0]; + + foreach my $comp2 (@$components2) { + my $diff_id = $comp2->{diff_id}; + # print "$diff_id\n"; + + #} + + #my $diff_id = 60382; + + if (!defined($diff_id)) { + die "Unable to parse diff_id"; + + } + + +# Database connection +my $db = DBI->connect( "DBI:mysql:database=$dbname;host=$db_host;mysql_socket=" . DB_SOCKET(), + $db_user, + $db_pw, + { RaiseError => 1, AutoCommit => 1 } + ) or die "Unable to connect to database: $DBI::errstr"; + +# Query to run +my $sql = "SELECT path_base, diffSkyfile.diff_id, skycell_id, diffSkyfile.bg,diffSkyfile.bg_stdev,diffSkyfile.dtime_diff,diffSkyfile.dtime_match,diffSkyfile.dtime_phot,diffSkyfile.dtime_script,diffSkyfile.stamps_num,diffSkyfile.stamps_mean,diffSkyfile.stamps_rms,diffSkyfile.sources from diffRun, diffSkyfile WHERE diffRun.diff_id = diffSkyfile.diff_id AND diffSkyfile.diff_id = $diff_id"; + +#print "$sql\n"; + +my $results = $db->selectall_arrayref( $sql ) or die "Unable to execute SQL: $DBI::errstr"; +$db->disconnect; + +foreach my $row ( @$results ) { + my $path_base = $$row[0]; # Chip path + my $diff_id1 = $$row[1]; # Class identifier + my $skycell_id = $$row[2]; # Camera path + + my $logfile = $path_base.".log"; + + if (defined($diffandsky)) { + +print "$diff_id1 $skycell_id "; + } else { +print "$logfile "; + } + + + if (defined($ippMonitor)) { + + if (defined($$row[3])) {printf "%2.4f ",$$row[3];} else {print "??? ";} + if (defined($$row[4])) {printf "%2.4f ",$$row[4];} else {print "??? ";} +if (defined($$row[5])) {printf "%2.4f ",$$row[5];} else {print "??? ";} +if (defined($$row[6])) {printf "%2.4f ",$$row[6];} else {print "??? ";} +if (defined($$row[7])) {printf "%2.4f ",$$row[7];} else {print "??? ";} +if (defined($$row[8])) {printf "%d ",$$row[8];} else {print "??? ";} +if (defined($$row[9])) {printf "%d ",$$row[9];} else {print "??? ";} +if (defined($$row[10])) {printf "%2.4f ",$$row[10];} else {print "??? ";} +if (defined($$row[11])) {printf "%2.4f ",$$row[11];} else {print "??? ";} +if (defined($$row[12])) {printf "%d ",$$row[12];} else {print "??? ";} + + + + + +# $$row[4] $$row[5] $$row[6] $$row[7] $$row[8] $$row[9] $$row[10] $$row[11] $$row[12] "; + + } + + #print "$logfile "; + if (defined($grep)) { + my $comm = "$neblocate --path $logfile --server http://nebulous.mhpcc.ipp.ifa.hawaii.edu:80/nebulous"; + + my ( $nebsuccess, $neberror_code, $nebfull_buf, $nebstdout_buf, $nebstderr_buf ) = + run(command => $comm); + + if (defined($nebstdout_buf)) { + my $grepcomm = "grep '$grep' $$nebstdout_buf[0]"; + # print "$grepcomm\n"; + my ( $grepsuccess, $greperror_code, $grepfull_buf, $grepstdout_buf, $grepstderr_buf ) = + run(command => $grepcomm); + + foreach my $grepout (@$grepstdout_buf) { + print $grepout; + + } + + } + + + } + + + } + +} + + +} + +#} + +# Pau. + Index: /branches/eam_branches/ipp-20100621/tools/examine_burntool_pcontrol.pl =================================================================== --- /branches/eam_branches/ipp-20100621/tools/examine_burntool_pcontrol.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/examine_burntool_pcontrol.pl (revision 28980) @@ -33,5 +33,5 @@ while() { chomp; - if ($_ =~ /BURNTOOL.STATE.GOOD/) { + if ($_ =~ /BURNTOOL.STATE.GOOD\s/) { @line = split /\s+/; $burntoolStateGood = $line[2]; Index: /branches/eam_branches/ipp-20100621/tools/psstatus =================================================================== --- /branches/eam_branches/ipp-20100621/tools/psstatus (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/psstatus (revision 28980) @@ -10,6 +10,7 @@ use File::Temp qw(tempfile); -my $limit = 20; +my $limit = 0; my $running; +my $finished; my $req_faulted; my $job_faulted; @@ -23,4 +24,5 @@ GetOptions( 'running|r', \$running, + 'finished|f', \$finished, 'req_faulted', \$req_faulted, 'job_faulted', \$job_faulted, @@ -32,10 +34,14 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -my $no_args = ! (defined $running or defined $req_faulted or defined $job_faulted ); +my $no_args = ! (defined $running or defined $finished or defined $req_faulted or defined $job_faulted ); + +die "cannot supply --running and --finished\n" if ($running and $finished); + my $ipprc = PS::IPP::Config->new(); my $dbh = getDBHandle(); -my $select = "SELECT req_id, name, label, reqType, pstampRequest.state, pstampRequest.fault, count(job_id) AS numJobs, timestamp"; +my $jobLabel = $running ? "Running jobs" : "Num Jobs"; +my $select = "SELECT req_id, name, label, reqType, pstampRequest.state, pstampRequest.fault, count(job_id) AS '$jobLabel', timestamp"; $select .= ", outdir" if ($verbose); @@ -45,8 +51,16 @@ my $sql = $select . $from . $where; -if ($no_args and $limit) { +$limit = 20 if ($no_args); +# if ($no_args and $limit) { +if (0 and $limit) { # use mysql to get the last_req_id # this works but it prints out the @last_req_id value which is distracting. Use dbi for the query - my $stmt = $dbh->prepare("SELECT req_id FROM pstampRequest ORDER BY req_id DESC LIMIT 1"); + my $where = ""; + if ($running) { + $where = " WHERE pstampRequest.state = 'run' OR pstampRequest.state = 'new'"; + } elsif ($finished) { + $where = " WHERE pstampRequest.state = 'stop'"; + } + my $stmt = $dbh->prepare("SELECT req_id FROM pstampRequest $where ORDER BY req_id DESC LIMIT 1"); $stmt->execute(); my $row = $stmt->fetchrow_hashref(); @@ -59,5 +73,7 @@ } if ($running) { - $sql .= " AND (pstampRequest.state = 'new' OR pstampRequest.state = 'run')" + $sql .= " AND (pstampRequest.state = 'new' OR pstampRequest.state = 'run') AND (pstampJob.state = 'run' OR pstampJob.state IS NULL)"; +} elsif ($finished) { + $sql .= " AND (pstampRequest.state = 'stop') AND date_add(timestamp, interval 1 day) >=utc_timestamp()" } if ($req_faulted) { @@ -69,4 +85,7 @@ $sql .= " GROUP BY req_id"; +if ($finished) { + $sql .= " ORDER BY timestamp DESC"; +} if ($limit and !$no_args) { $sql .= " LIMIT $limit"; Index: /branches/eam_branches/ipp-20100621/tools/roboczar.pl =================================================================== --- /branches/eam_branches/ipp-20100621/tools/roboczar.pl (revision 28979) +++ /branches/eam_branches/ipp-20100621/tools/roboczar.pl (revision 28980) @@ -10,6 +10,6 @@ use czartool::Gpc1Db; use czartool::Pantasks; +use czartool::Nebulous; use czartool::Czarplot; - my $period = 60; @@ -24,6 +24,7 @@ my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser"); +my $nebulous = new czartool::Nebulous($czarDb); my $pantasks = new czartool::Pantasks(); -my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", "png", "/tmp", $save_temps); +my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path $czarDb->setDateFormat("%Y%m%d-%H%i%s"); @@ -41,5 +42,5 @@ print "* Updating labels\n"; - my @servers = ("stdscience", "distribution", "publishing"); + my @servers = ("stdscience", "distribution", "publishing", "update"); my $server = undef; @@ -91,34 +92,15 @@ my $faults; my $stage; - my ($totalNew,$totalFull,$totalFaults); my $query = undef; my $str = undef; - my $reverting = 0; - my $server = undef; - my $rows = undef; + my $labels = undef; + my $updateLabels = undef; my $row = undef; my $begin = undef; my $end = undef; my $priority = undef; + my $newState = undef; while (1) { - - updateServerStatus(); - updateLabels(); - if (!$czarDb->getCurrentLabels("stdscience", \$rows)) {next;} - - my $size = @{$rows}; - if($size < 1) { - - print "* WARNING: no stdscience labels found in Db\n"; - next; - } - - # get priority - foreach $row ( @{$rows} ) { - my ($label) = @{$row}; - $priority = $gpc1Db->getPriority($label); - $czarDb->setLabelPriority($label, $priority); - } # sort out times @@ -131,49 +113,40 @@ } - foreach $stage (@stages) { - - $server = $pantasks->getServerForThisStage($stage); - $str = `czartool_revert.pl -s $server -t $stage`; - if ($str =~ m/on/) {$reverting = 1;} - else {$reverting = 0;} - print "* Checking labels for $stage stage\n"; - - $totalNew=$totalFaults=$totalFull=0; - foreach $row ( @{$rows} ) { - my ($label) = @{$row}; - - chomp($label); - - $new = $gpc1Db->countExposures($label, $stage, "new"); - $full = $gpc1Db->countExposures($label, $stage, "full"); - $faults = $gpc1Db->countFaults($label, $stage); - - #printf("%ld, %s, %s, %d, %d\n", $currentTime, $label, $stage, $new, $faults); - $totalNew += $new; - $totalFull += $full; - $totalFaults += $faults; - - $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults, $reverting); - $czarplot->createTimeSeries($label, $stage, $begin, $end); + # check nebulous + print "* Checking Nebulous\n"; + $nebulous->updateClusterSpaceInfo(); + $czarplot->plotDiskUsageHistogram(); + updateServerStatus(); + + # check labels + updateLabels(); + + # servers to check + my @serversToCheck = ("stdscience", "update"); + + my $thisServer = undef; + foreach $thisServer (@serversToCheck) { + + if ($thisServer eq "update") {$newState = "update";} + else {$newState = "new";} + + # deal with stdscience labels + if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;} + my $size = @{$labels}; + if($size > 0) { + + # get priority + foreach $row ( @{$labels} ) { + my ($label) = @{$row}; + $priority = $gpc1Db->getPriority($label); + $czarDb->setLabelPriority($label, $priority); + } + + updateAllStages($thisServer, $newState, $labels, $begin, $end); + createPlots($thisServer, $labels, $begin, $end); } - - $czarDb->insertNewTimeData($stage, "all_labels", $totalNew, $totalFull, $totalFaults, $reverting); - } - - print "* Generating plots\n"; - foreach $row ( @{$rows} ) { - my ($label) = @{$row}; - - $czarplot->createTimeSeries($label, undef, $begin, $end); - $czarplot->createHistogram($label, $begin, $end); - - #routineChecks($label, "1 HOUR"); - } - $czarplot->createTimeSeries("all_labels", undef, $begin, $end); - $czarplot->createHistogram("all_labels", $begin, $end); - foreach $stage (@stages) { - - $czarplot->createTimeSeries("all_labels", $stage, $begin, $end); # TODO must be a neater way... - } + else { print "* WARNING: no $thisServer labels found in Db\n";} + } + print "--------------------------------------------------------------------------\n"; print "* Going to sleep\n"; @@ -183,4 +156,95 @@ #sendEmail("roydhenderson\@gmail.com", "roboczar\@ipp.com", "Roboczar update", "Some content"); }; +} + +########################################################################### +# +# Loops through labels and creates time series and histogram plots +# +########################################################################### +sub createPlots { + my ($server, $rows, $begin, $end) = @_; + + my $stage = undef; + my $row = undef; + + print "* Generating plots\n"; + + # create plots for each label for each stage + foreach $stage (@stages) { + foreach $row ( @{$rows} ) { + my ($label) = @{$row}; + + chomp($label); + $czarplot->createTimeSeries($label, $stage, $begin, $end); + } + } + + # create plots for each label for all stages + foreach $row ( @{$rows} ) { + my ($label) = @{$row}; + + $czarplot->createTimeSeries($label, undef, $begin, $end); + $czarplot->createHistogram($label, $begin, $end); + + #routineChecks($label, "1 HOUR"); + } + $czarplot->createTimeSeries("all_".$server."_labels", undef, $begin, $end); + $czarplot->createHistogram("all_".$server."_labels", $begin, $end); + foreach $stage (@stages) { + + $czarplot->createTimeSeries("all_".$server."_labels", $stage, $begin, $end); # TODO must be a neater way... + } +} + +########################################################################### +# +# Loops through some labels and updates processed/pending/faults in the Db +# +########################################################################### +sub updateAllStages { + my ($labelServer, $newState, $rows, $begin, $end) = @_; + + print "* Updating stage data\n"; + my $totalNew = undef; + my $totalFaults = undef; + my $totalFull = undef; + my $stage = undef; + my $reverting = 0; + my $row = undef; + my $new = undef; + my $full = undef; + my $faults = undef; + my $server = undef; + my $state = undef; + + foreach $stage (@stages) { + + $server = $pantasks->getServerForThisStage($stage); + $pantasks->getRevertStatus($stage, \$reverting); + $czarDb->updateRevertStatus($stage, $reverting); + + print "* Checking labels for $stage stage\n"; + + $totalNew=$totalFaults=$totalFull=0; + foreach $row ( @{$rows} ) { + my ($label) = @{$row}; + + chomp($label); + + $new = $gpc1Db->countExposures($label, $stage, $newState); + $full = $gpc1Db->countExposures($label, $stage, "full"); + $faults = $gpc1Db->countFaults($label, $stage, $newState); + + #printf("%s %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults); + $totalNew += $new; + $totalFull += $full; + $totalFaults += $faults; + + $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults); + } + + $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", $totalNew, $totalFull, $totalFaults); + } }