Index: /tags/ipp-20100823/dbconfig/background.md =================================================================== --- /tags/ipp-20100823/dbconfig/background.md (revision 29497) +++ /tags/ipp-20100823/dbconfig/background.md (revision 29498) @@ -21,4 +21,5 @@ class_id STR 64 path_base STR 255 + data_state STR 64 magicked S64 0 dtime_script F32 0.0 @@ -57,4 +58,5 @@ skycell_id STR 64 path_base STR 255 + data_state STR 64 magicked S64 0 dtime_script F32 0.0 Index: /tags/ipp-20100823/dbconfig/changes.txt =================================================================== --- /tags/ipp-20100823/dbconfig/changes.txt (revision 29497) +++ /tags/ipp-20100823/dbconfig/changes.txt (revision 29498) @@ -1945,2 +1945,20 @@ ALTER TABLE diffSummary DROP PRIMARY KEY, ADD PRIMARY KEY (diff_id, projection_cell); ALTER TABLE stackSummary DROP PRIMARY KEY, ADD PRIMARY KEY (sass_id, projection_cell); + +ALTER TABLE stackSumSkyfile ADD COLUMN mjd_obs double AFTER good_frac; + +-- changes to support cleanup of the background preserved images +ALTER TABLE chipBackgroundImfile ADD COLUMN data_state VARCHAR(64) AFTER path_base; +ALTER TABLE chipBackgroundImfile ADD KEY(data_state); +ALTER TABLE warpBackgroundSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base; +ALTER TABLE warpBackgroundSkyfile ADD KEY(data_state); + +ALTER TABLE magicMask ADD COLUMN path_base VARCHAR(255) AFTER uri; + +-- diff_id was removed from magicInputSkyfile ages ago, but +-- pxadmin_create_tables.sql still had it this command didn't work due to +-- a mysql error. +-- ALTER TABLE magicInputSkyfile DROP COLUMN diff_id; +UPDATE dbversion set schema_version = '1.1.64', updated= CURRENT_TIMESTAMP(); + +-- Version 1.1.65 Index: /tags/ipp-20100823/dbconfig/config.md =================================================================== --- /tags/ipp-20100823/dbconfig/config.md (revision 29497) +++ /tags/ipp-20100823/dbconfig/config.md (revision 29498) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.63 + pkg_version STR 1.1.64 END Index: /tags/ipp-20100823/dbconfig/magic.md =================================================================== --- /tags/ipp-20100823/dbconfig/magic.md (revision 29497) +++ /tags/ipp-20100823/dbconfig/magic.md (revision 29498) @@ -42,4 +42,5 @@ magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) uri STR 255 + path_base STR 255 streaks S32 0 fault S16 0 # Key Index: /tags/ipp-20100823/ippTasks/destreak.pro =================================================================== --- /tags/ipp-20100823/ippTasks/destreak.pro (revision 29497) +++ /tags/ipp-20100823/ippTasks/destreak.pro (revision 29498) @@ -29,4 +29,13 @@ $magicDSRevertStage = 0 +macro destreak.show.stages + echo $DS_STAGE:n stages enabled for destreak processing + for i 0 $DS_STAGE:n + echo $i $DS_STAGE:$i + end + echo current magicDSStage = $magicDSStage + echo current magicDSRevertStage = $magicDSRevertStage +end + ### Check status of tasks macro destreak.status @@ -178,5 +187,7 @@ book getword magicToDS $pageName camera -var CAMERA book getword magicToDS $pageName streaks_uri -var STREAKS + book getword magicToDS $pageName streaks_path_base -var STREAKS_PATH_BASE book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS + book getword magicToDS $pageName inv_streaks_path_base -var INV_STREAKS_PATH_BASE book getword magicToDS $pageName stage -var STAGE book getword magicToDS $pageName stage_id -var STAGE_ID @@ -204,5 +215,5 @@ # TODO: do not add recoveryroot or replace if they are null or zero - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE--streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED add_standard_args run @@ -249,6 +260,5 @@ periods -poll $LOADPOLL - #periods -exec $LOADEXEC - periods -exec 60 + periods -exec 30 periods -timeout 300 npending 1 Index: /tags/ipp-20100823/ippTasks/diskbalance.pro =================================================================== --- /tags/ipp-20100823/ippTasks/diskbalance.pro (revision 29498) +++ /tags/ipp-20100823/ippTasks/diskbalance.pro (revision 29498) @@ -0,0 +1,229 @@ +## diskbalance.pro : tasks for data replication : -*- sh -*- +## this file contains the tasks for maintaining duplicates as needed in Nebulous +## these tasks use the books balancePending + +# test for required global variables +check.globals + +# load in the local configuration (NEB_USER, etc.) +module nebulous.site.pro + +$LOGSUBDIR = $LOGDIR/balance +mkdir $LOGSUBDIR + +book init balancePending +book init balanceControl + +macro balance.reset + book init balancePending +end + +macro balance.status + book listbook balancePending +end + +macro balance.on + task balance.govern + active true + end + task balance.load + active true + end + task balance.run + active true + end +end + +macro balance.off + task balance.govern + active false + end + task balance.load + active false + end + task balance.run + active false + end +end + +macro set.host.for.balance + if ($0 != 2) + echo "USAGE: set.host.for.balance (hostname)" + break + end + + if (not($PARALLEL)) + host local + return + end + +# parse volume name + + if ("$1" == "NULL") + host anyhost + else + host $1 + end +end + +macro set.balance.numbers + if ($0 != 3) + echo "USAGE: set.balance.numbers (N_sources) (M_destinations)" + break + end + + book newpage balanceControl control + book setword balanceControl control nSources $1 + book setword balanceControl control mDestinations $2 +end + +# the balance process interacts with only the single Nebulous server + +# Each 'pendingbalance query is limited to a finite number of so_id values. +# Each time we call balance.load, we increment the so_id start by the range value +# If the pendingbalance query exits with exit status 10, we start over at 0 + +$BALANCE_SO_ID_START = 0 +$BALANCE_SO_ID_RANGE = 500000 + +# Select Nebulous objects which should be shuffled +task balance.load + host local + + periods -poll 0.5 + periods -exec 5 + periods -timeout 1500 + npending 1 + + # logs + stdout NULL + stderr $LOGSUBDIR/balance.log + + task.exec + book npages balancePending -var N + if ($N > 2000) + process_cleanup balancePending + break + end + + book getword balanceControl control nSources -var BALANCE_N + book getword balanceControl control mDestinations -var BALANCE_M + + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingbalance --limit 500 --so_id_start $BALANCE_SO_ID_START --so_id_range $BALANCE_SO_ID_RANGE --balancesources $BALANCE_N --balancedestination $BALANCE_M + + end + + # success : 0 -- we did not hit the limit, advance so_id counter + task.exit 0 + # advance the so_id counter + $BALANCE_SO_ID_START = $BALANCE_SO_ID_START + $BALANCE_SO_ID_RANGE + + # convert 'stdout' to book format + ipptool2book stdout balancePending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook balancePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup balancePending + end + + # success : 1 -- we DID hit the limit, do NOT advance so_id counter + task.exit 1 + # convert 'stdout' to book format + ipptool2book stdout balancePending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook balancePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup balancePending + end + + # out of so_id range, reset + task.exit 10 + # advance the so_id counter + $BALANCE_SO_ID_START = 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# task to execute shuffles +task balance.run + periods -poll 0.5 + periods -exec 5 + periods -timeout 30 + + task.exec + book npages balancePending -var N + if ($NETWORK == 0) break + if ($N == 0) + periods -exec 5 + break + end + periods -exec 0.05 + + # look for new objects in balancePending + book getpage balancePending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setwork balancePending $pageName pantaskState RUN + + book getword balancePending $pageName key -var KEY + book getword balancePending $pageName source_name -var SOURCE + book getword balancePending $pageName destination_name -var DESTINATION + + # Fix this with multihost restriction when possible. + set.host.for.balance $DESTINATION + + stdout NULL + stderr $LOGSUBDIR/balance.log + + $run = neb-shift --volume $DESTINATION $KEY $SOURCE + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit balancePending $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword balancePending $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword balancePending $options:0 pantaskState TIMEOUT + end +end + + + + + Index: /tags/ipp-20100823/ippTasks/magic.pro =================================================================== --- /tags/ipp-20100823/ippTasks/magic.pro (revision 29497) +++ /tags/ipp-20100823/ippTasks/magic.pro (revision 29498) @@ -288,4 +288,6 @@ book getword magicToProcess $pageName camera -var CAMERA book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE + book getword magicToProcess $pageName raw_workdir -var RAW_TEMPLATE + book getword magicToProcess $pageName exp_name -var EXP_NAME book getword magicToProcess $pageName dbname -var DBNAME @@ -308,4 +310,15 @@ sprintf baseroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + if ("$NODE" == "root") + # This is the root node, put the streaks file into the rawExps workdir + # which is presumably in nebulous and thus will be replicatable + set.workdir.by.camera $CAMERA FPA $RAW_TEMPLATE $default_host RAW_WORKDIR + sprintf final_outroot "%s/%s.%s/%s.%s.mgc.%s" $RAW_WORKDIR $EXP_NAME $EXP_ID $EXP_NAME $EXP_ID $MAGIC_ID + + $EXTRA_ARGS = --final-outroot $final_outroot + else + $EXTRA_ARGS = "" + end + $logfile = $baseroot.$NODE.log if ("$logfile" == "") @@ -319,5 +332,5 @@ mkdir $outpath - $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile + $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile $EXTRA_ARGS add_standard_args run Index: /tags/ipp-20100823/ippTasks/replicate.pro =================================================================== --- /tags/ipp-20100823/ippTasks/replicate.pro (revision 29497) +++ /tags/ipp-20100823/ippTasks/replicate.pro (revision 29498) @@ -173,4 +173,5 @@ book getword replicatePending $pageName volume_name -var VOLUME_NAME book getword replicatePending $pageName volume_host -var VOLUME_HOST + book getword replicatePending $pageName command -var COMMAND set.host.for.replicate $VOLUME_HOST @@ -180,5 +181,5 @@ # these operations do not require a database to be specified - $run = neb-replicate --copies $NEED_COPIES $KEY + $run = $COMMAND $KEY # save the pageName for future reference below Index: /tags/ipp-20100823/ippTasks/science.cleanup.pro =================================================================== --- /tags/ipp-20100823/ippTasks/science.cleanup.pro (revision 29497) +++ /tags/ipp-20100823/ippTasks/science.cleanup.pro (revision 29498) @@ -113,5 +113,8 @@ task.exec book npages chipPendingCleanup -var N - if ($N == 0) break + if ($N == 0) + periods -exec $RUNEXEC + break + end if ($NETWORK == 0) break @@ -143,4 +146,5 @@ echo command $run end + periods -exec 0.05 command $run end @@ -569,5 +573,8 @@ task.exec book npages warpPendingCleanup -var N - if ($N == 0) break + if ($N == 0) + periods -exec $RUNEXEC + break + end if ($NETWORK == 0) break @@ -599,4 +606,5 @@ echo command $run end + periods -exec 0.05 command $run end @@ -721,5 +729,8 @@ task.exec book npages diffCleanup -var N - if ($N == 0) break + if ($N == 0) + periods -exec $RUNEXEC + break + end if ($NETWORK == 0) break @@ -751,4 +762,5 @@ echo command $run end + periods -exec 0.05 command $run end Index: /tags/ipp-20100823/ippTools/configure.ac =================================================================== --- /tags/ipp-20100823/ippTools/configure.ac (revision 29497) +++ /tags/ipp-20100823/ippTools/configure.ac (revision 29498) @@ -1,5 +1,5 @@ AC_PREREQ(2.61) -AC_INIT([ipptools], [1.1.63], [ipp-support@ifa.hawaii.edu]) +AC_INIT([ipptools], [1.1.64], [ipp-support@ifa.hawaii.edu]) AC_CONFIG_SRCDIR([autogen.sh]) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.64]) PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) Index: /tags/ipp-20100823/ippTools/share/Makefile.am =================================================================== --- /tags/ipp-20100823/ippTools/share/Makefile.am (revision 29497) +++ /tags/ipp-20100823/ippTools/share/Makefile.am (revision 29498) @@ -61,4 +61,5 @@ chiptool_pendingimfile.sql \ chiptool_processedimfile.sql \ + chiptool_revertcleanup.sql \ chiptool_revertprocessedimfile.sql \ chiptool_revertupdatedimfile.sql \ @@ -143,4 +144,5 @@ difftool_revertdiffskyfile_delete.sql \ difftool_revertdiffskyfile_updated.sql \ + difftool_revertcleanup.sql \ difftool_setskyfiletoupdate.sql \ difftool_skyfile.sql \ @@ -358,4 +360,5 @@ warptool_revertwarped_delete.sql \ warptool_revertwarped_updated.sql \ + warptool_revertcleanup.sql \ warptool_runstate.sql \ warptool_scmap.sql \ Index: /tags/ipp-20100823/ippTools/share/addtool_find_cam_id.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/addtool_find_cam_id.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/addtool_find_cam_id.sql (revision 29498) @@ -2,4 +2,6 @@ camRun.* FROM camRun +JOIN camProcessedExp + USING(cam_id) JOIN chipRun USING(chip_id) @@ -10,5 +12,5 @@ FROM addRun JOIN camRun USING(cam_id) - JOIN chipRun USING(chip_id) + JOIN chipRun USING(chip_id) ) as foo ON exp_id = added_exp_id @@ -17,4 +19,5 @@ WHERE camRun.state = 'full' + AND camProcessedExp.quality = 0 AND added_exp_id IS NULL -- addtool adds checks on exposure being added to the dvodb previously Index: /tags/ipp-20100823/ippTools/share/addtool_find_cam_id_dvo.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/addtool_find_cam_id_dvo.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/addtool_find_cam_id_dvo.sql (revision 29498) @@ -1,6 +1,7 @@ SELECT camRun.* FROM camRun +JOIN camProcessedExp USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) -WHERE camRun.state = 'full' +WHERE camRun.state = 'full' and camProcessedExp.quality = 0 AND exp_id NOT IN (SELECT exp_id FROM addRun Index: /tags/ipp-20100823/ippTools/share/chiptool_revertcleanup.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/chiptool_revertcleanup.sql (revision 29498) +++ /tags/ipp-20100823/ippTools/share/chiptool_revertcleanup.sql (revision 29498) @@ -0,0 +1,7 @@ +UPDATE chipProcessedImfile + JOIN chipRun using(chip_id,exp_id) +SET chipProcessedImfile.data_state = 'full', + chipRun.state = '%s' +WHERE + chipRun.state = '%s' + AND chipProcessedImfile.data_state = '%s' Index: /tags/ipp-20100823/ippTools/share/difftool_revertcleanup.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/difftool_revertcleanup.sql (revision 29498) +++ /tags/ipp-20100823/ippTools/share/difftool_revertcleanup.sql (revision 29498) @@ -0,0 +1,7 @@ +UPDATE diffSkyfile + JOIN diffRun using(diff_id) +SET diffSkyfile.data_state = 'full', + diffRun.state = '%s' +WHERE + diffRun.state = '%s' + AND diffSkyfile.data_state = '%s' Index: /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_camera.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_camera.sql (revision 29498) @@ -7,5 +7,7 @@ camera, magicMask.uri AS streaks_uri, + magicMask.path_base AS streaks_path_base, CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, stage, stage_id, Index: /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_chip.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_chip.sql (revision 29498) @@ -7,5 +7,7 @@ camera, magicMask.uri AS streaks_uri, + magicMask.path_base AS streaks_path_base, CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, stage, stage_id, Index: /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_diff.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_diff.sql (revision 29498) @@ -7,5 +7,7 @@ rawExp.camera, magicMask.uri AS streaks_uri, + magicMask.path_base AS streaks_path_base, CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, stage, magicRun.diff_id AS stage_id, @@ -52,5 +54,7 @@ rawExp.camera, magicMask.uri AS streaks_uri, + magicMask.path_base AS streaks_path_base, (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, + (SELECT path_base from magicMask where magic_id = inv_magic_id) AS inv_streaks_path_base, stage, magicRun.diff_id AS stage_id, Index: /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_raw.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_raw.sql (revision 29498) @@ -6,5 +6,7 @@ rawExp.camera, magicMask.uri as streaks_uri, + magicMask.path_base as streaks_path_base, CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, stage, stage_id, Index: /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_warp.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/magicdstool_todestreak_warp.sql (revision 29498) @@ -6,5 +6,7 @@ camera, magicMask.uri as streaks_uri, + magicMask.path_base as streaks_path_base, CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_path_base, stage, stage_id, Index: /tags/ipp-20100823/ippTools/share/magictool_toprocess_inputs.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/magictool_toprocess_inputs.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/magictool_toprocess_inputs.sql (revision 29498) @@ -3,4 +3,6 @@ magicRun.workdir, rawExp.exp_id, + rawExp.exp_name, + rawExp.workdir AS raw_workdir, rawExp.camera, -- convert magic_id into a boolean value (1 or 0) Index: /tags/ipp-20100823/ippTools/share/magictool_toprocess_tree.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/magictool_toprocess_tree.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/magictool_toprocess_tree.sql (revision 29498) @@ -3,4 +3,6 @@ magicRun.workdir, rawExp.exp_id, + rawExp.exp_name, + rawExp.workdir AS raw_workdir, rawExp.camera, -- convert magic_id into a boolean value (1 or 0) Index: /tags/ipp-20100823/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/pxadmin_create_tables.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/pxadmin_create_tables.sql (revision 29498) @@ -1221,9 +1221,7 @@ CREATE TABLE magicInputSkyfile ( magic_id BIGINT, - diff_id BIGINT, node VARCHAR(64), - PRIMARY KEY(magic_id, diff_id, node), - FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), - FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) + PRIMARY KEY(magic_id, node), + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1253,4 +1251,5 @@ magic_id BIGINT, uri VARCHAR(255), + path_base VARCHAR(255), streaks INT, fault SMALLINT, Index: /tags/ipp-20100823/ippTools/share/stacktool_tosum.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/stacktool_tosum.sql (revision 29497) +++ /tags/ipp-20100823/ippTools/share/stacktool_tosum.sql (revision 29498) @@ -7,12 +7,15 @@ stackRun.label, stackRun.state, - stackSumSkyfile.path_base + stackSumSkyfile.path_base, + IFNULL(Label.priority, 10000) AS priority FROM stackRun JOIN stackInputSkyfile USING(stack_id) JOIN warpRun USING(warp_id) LEFT JOIN stackSumSkyfile USING(stack_id) +LEFT JOIN Label ON Label.label = stackRun.label WHERE ((stackRun.state = 'new' AND stackSumSkyfile.stack_id IS NULL) OR (stackRun.state = 'update' AND stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0)) + AND (Label.active OR Label.active IS NULL) -- WHERE hook %s GROUP BY stack_id Index: /tags/ipp-20100823/ippTools/share/warptool_revertcleanup.sql =================================================================== --- /tags/ipp-20100823/ippTools/share/warptool_revertcleanup.sql (revision 29498) +++ /tags/ipp-20100823/ippTools/share/warptool_revertcleanup.sql (revision 29498) @@ -0,0 +1,7 @@ +UPDATE warpSkyfile + JOIN warpRun using(warp_id) +SET warpSkyfile.data_state = 'full', + warpRun.state = '%s' +WHERE + warpRun.state = '%s' + AND warpSkyfile.data_state = '%s' Index: /tags/ipp-20100823/ippTools/src/addtool.c =================================================================== --- /tags/ipp-20100823/ippTools/src/addtool.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/addtool.c (revision 29498) @@ -493,9 +493,13 @@ // since there is only one exp per 'new' set addRun.state = 'full' - if (!pxaddRunSetState(config, row->add_id, "full")) { + // but check to make sure there are no faults + + if (!fault) { + if (!pxaddRunSetState(config, row->add_id, "full")) { psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); psFree(row); psFree(pendingRow); return false; + } } psFree(row); Index: /tags/ipp-20100823/ippTools/src/bgtool.c =================================================================== --- /tags/ipp-20100823/ippTools/src/bgtool.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/bgtool.c (revision 29498) @@ -658,5 +658,5 @@ } - if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script, + if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, "full", magicked, dtime_script, hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { @@ -1497,5 +1497,5 @@ } - if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script, + if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, "full", magicked, dtime_script, hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { Index: /tags/ipp-20100823/ippTools/src/chiptool.c =================================================================== --- /tags/ipp-20100823/ippTools/src/chiptool.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/chiptool.c (revision 29498) @@ -50,4 +50,5 @@ static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupimfileMode(pxConfig *config); +static bool revertcleanupMode(pxConfig *config); static bool donecleanupMode(pxConfig *config); static bool runMode(pxConfig *config); @@ -94,4 +95,5 @@ MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileMode); + MODECASE(CHIPTOOL_MODE_REVERTCLEANUP, revertcleanupMode); MODECASE(CHIPTOOL_MODE_DONECLEANUP, donecleanupMode); MODECASE(CHIPTOOL_MODE_RUN, runMode); @@ -859,9 +861,10 @@ } psString query_update = pxDataGet("chiptool_revertupdatedimfile.sql"); - if (!query) { + if (!query_update) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; } + if (where && psListLength(where->list)) { @@ -886,4 +889,59 @@ } psFree(query_update); + + return true; +} +static bool revertcleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + char* newState = NULL; + if (!state) { + state = "error_cleaned"; + } + if (!strcmp(state, "error_cleaned")) { + newState = "goto_cleaned"; + } else if (!strcmp(state, "error_purged")) { + newState = "goto_purged"; + } else if (!strcmp(state, "error_scrubbed")) { + newState = "goto_scrubbed"; + } else { + psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("chiptool_revertcleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); return true; Index: /tags/ipp-20100823/ippTools/src/chiptool.h =================================================================== --- /tags/ipp-20100823/ippTools/src/chiptool.h (revision 29497) +++ /tags/ipp-20100823/ippTools/src/chiptool.h (revision 29498) @@ -42,4 +42,5 @@ CHIPTOOL_MODE_PENDINGCLEANUPRUN, CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, + CHIPTOOL_MODE_REVERTCLEANUP, CHIPTOOL_MODE_DONECLEANUP, CHIPTOOL_MODE_RUN, Index: /tags/ipp-20100823/ippTools/src/chiptoolConfig.c =================================================================== --- /tags/ipp-20100823/ippTools/src/chiptoolConfig.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/chiptoolConfig.c (revision 29498) @@ -290,4 +290,12 @@ psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -revertcleanup + psMetadata *revertcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state", 0, "search by current state", NULL); + psMetadataAddS16(revertcleanupArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + // -run psMetadata *runArgs = psMetadataAlloc(); @@ -371,4 +379,5 @@ PXOPT_ADD_MODE("-pendingcleanupimfile", "show runs that need to be cleaned up", CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileArgs); PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", CHIPTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-revertcleanup", "show runs that have been cleaned", CHIPTOOL_MODE_REVERTCLEANUP, revertcleanupArgs); PXOPT_ADD_MODE("-run", "show runs", CHIPTOOL_MODE_RUN, runArgs); PXOPT_ADD_MODE("-tocleanedimfile", "set imfile state to cleaned", CHIPTOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileArgs); Index: /tags/ipp-20100823/ippTools/src/difftool.c =================================================================== --- /tags/ipp-20100823/ippTools/src/difftool.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/difftool.c (revision 29498) @@ -48,4 +48,5 @@ static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); +static bool revertcleanupMode(pxConfig *config); static bool donecleanupMode(pxConfig *config); static bool updatediffskyfileMode(pxConfig *config); @@ -100,4 +101,5 @@ MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); + MODECASE(DIFFTOOL_MODE_REVERTCLEANUP, revertcleanupMode); MODECASE(DIFFTOOL_MODE_DONECLEANUP, donecleanupMode); MODECASE(DIFFTOOL_MODE_UPDATEDIFFSKYFILE, updatediffskyfileMode); @@ -2531,4 +2533,61 @@ +static bool revertcleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + if (!state) { + state = "error_cleaned"; + } + char *newState = NULL; + if (!strcmp(state, "error_cleaned")) { + newState = "goto_cleaned"; + } else if (!strcmp(state, "error_purged")) { + newState = "goto_purged"; + } else if (!strcmp(state, "error_scrubbed")) { + newState = "goto_scrubbed"; + } else { + psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed"); + return false; + } + + psString query = pxDataGet("difftool_revertcleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); + + psLogMsg("difftool", PS_LOG_INFO, "Reverted %d diffRuns and diffSkyfiles", numDeleted); + + return true; +} static bool donecleanupMode(pxConfig *config) { Index: /tags/ipp-20100823/ippTools/src/difftool.h =================================================================== --- /tags/ipp-20100823/ippTools/src/difftool.h (revision 29497) +++ /tags/ipp-20100823/ippTools/src/difftool.h (revision 29498) @@ -42,4 +42,5 @@ DIFFTOOL_MODE_PENDINGCLEANUPRUN, DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, + DIFFTOOL_MODE_REVERTCLEANUP, DIFFTOOL_MODE_DONECLEANUP, DIFFTOOL_MODE_UPDATEDIFFSKYFILE, Index: /tags/ipp-20100823/ippTools/src/difftoolConfig.c =================================================================== --- /tags/ipp-20100823/ippTools/src/difftoolConfig.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/difftoolConfig.c (revision 29498) @@ -173,5 +173,5 @@ psMetadata *listrunArgs = psMetadataAlloc(); psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); - psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID (LIKE comparison)", NULL); psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", NULL); psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-template", 0, "apply exposure args to template of bothways diff", false); @@ -199,5 +199,5 @@ psMetadata *listssrunArgs = psMetadataAlloc(); psMetadataAddS64(listssrunArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); - psMetadataAddStr(listssrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(listssrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID (LIKE comparison)", NULL); psMetadataAddS64(listssrunArgs , PS_LIST_TAIL, "-stack_id", 0, "search by input stack_id", 0); psMetadataAddS64(listssrunArgs , PS_LIST_TAIL, "-template_stack_id",0, "search by template stack_id", 0); @@ -390,4 +390,12 @@ psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -revertcleanup + psMetadata *revertcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-diff_id", 0, "search by difftool ID", 0); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + + // -donecleanup psMetadata *donecleanupArgs = psMetadataAlloc(); @@ -464,4 +472,5 @@ PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); + PXOPT_ADD_MODE("-revertcleanup", "revert cleanup runs with errors", DIFFTOOL_MODE_REVERTCLEANUP, revertcleanupArgs); PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", DIFFTOOL_MODE_DONECLEANUP, donecleanupArgs); PXOPT_ADD_MODE("-updatediffskyfile", "update fault code for a diffskyfile", DIFFTOOL_MODE_UPDATEDIFFSKYFILE, updatediffskyfileArgs); Index: /tags/ipp-20100823/ippTools/src/magictool.c =================================================================== --- /tags/ipp-20100823/ippTools/src/magictool.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/magictool.c (revision 29498) @@ -50,5 +50,5 @@ static bool exposureMode(pxConfig *config); -static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); +static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString); static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename); @@ -131,5 +131,5 @@ psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else - PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, queryWhere, "-select_filter", "rawExp.filter", "=="); @@ -394,9 +394,20 @@ // required PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + + psString setString = NULL; + if (fault || clearfault) { + psStringAppend(&setString, ", fault = %d", fault); + } + if (note) { + psStringAppend(&setString, ", note = '%s'", note); + } if (state) { // set detRun.state to state - return setmagicRunState(config, magic_id, state); + return setmagicRunState(config, magic_id, state, setString); } @@ -1171,5 +1182,6 @@ // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); +// PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false); @@ -1184,5 +1196,6 @@ if (!magicMaskInsert(config->dbh, magic_id, - uri, + NULL, + path_base, streaks, fault @@ -1364,5 +1377,5 @@ } -static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state) +static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString) { PS_ASSERT_PTR_NON_NULL(state, false); @@ -1380,7 +1393,13 @@ return false; } - - char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_id)) { + psString query = NULL; + psStringAppend(&query, "UPDATE magicRun SET state = '%s'", state); + if (setString) { + psStringAppend(&query, setString); + } + psStringAppend(&query, " WHERE magic_id = %" PRId64, magic_id);; + +// char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64; + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for magic_id %" PRId64, magic_id); Index: /tags/ipp-20100823/ippTools/src/magictoolConfig.c =================================================================== --- /tags/ipp-20100823/ippTools/src/magictoolConfig.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/magictoolConfig.c (revision 29498) @@ -79,5 +79,8 @@ psMetadata *updaterunArgs = psMetadataAlloc(); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", NULL); // -addinputskyfile @@ -149,5 +152,5 @@ psMetadata *addmaskArgs = psMetadataAlloc(); psMetadataAddS64(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL); psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0); psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); Index: /tags/ipp-20100823/ippTools/src/stacktool.c =================================================================== --- /tags/ipp-20100823/ippTools/src/stacktool.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/stacktool.c (revision 29498) @@ -900,4 +900,6 @@ } psFree(where); + + psStringAppend(&query, "\nORDER by priority DESC, stack_id"); // treat limit == 0 as "no limit" Index: /tags/ipp-20100823/ippTools/src/warptool.c =================================================================== --- /tags/ipp-20100823/ippTools/src/warptool.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/warptool.c (revision 29498) @@ -52,4 +52,5 @@ static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupwarpMode(pxConfig *config); +static bool revertcleanupMode(pxConfig *config); static bool donecleanupMode(pxConfig *config); static bool tocleanedskyfileMode(pxConfig *config); @@ -107,4 +108,5 @@ MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode); + MODECASE(WARPTOOL_MODE_REVERTCLEANUP, revertcleanupMode); MODECASE(WARPTOOL_MODE_DONECLEANUP, donecleanupMode); MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); @@ -1805,4 +1807,61 @@ +static bool revertcleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + if (!state) { + state = "error_cleaned"; + } + char *newState = NULL; + if (!strcmp(state, "error_cleaned")) { + newState = "goto_cleaned"; + } else if (!strcmp(state, "error_purged")) { + newState = "goto_purged"; + } else if (!strcmp(state, "error_scrubbed")) { + newState = "goto_scrubbed"; + } else { + psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed"); + return false; + } + + psString query = pxDataGet("warptool_revertcleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); + + psLogMsg("warptool", PS_LOG_INFO, "Reverted %d warpRuns and warpSkyfiles", numDeleted); + + return true; +} static bool donecleanupMode(pxConfig *config) { Index: /tags/ipp-20100823/ippTools/src/warptool.h =================================================================== --- /tags/ipp-20100823/ippTools/src/warptool.h (revision 29497) +++ /tags/ipp-20100823/ippTools/src/warptool.h (revision 29498) @@ -48,4 +48,5 @@ WARPTOOL_MODE_PENDINGCLEANUPRUN, WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, + WARPTOOL_MODE_REVERTCLEANUP, WARPTOOL_MODE_DONECLEANUP, WARPTOOL_MODE_TOCLEANEDSKYFILE, Index: /tags/ipp-20100823/ippTools/src/warptoolConfig.c =================================================================== --- /tags/ipp-20100823/ippTools/src/warptoolConfig.c (revision 29497) +++ /tags/ipp-20100823/ippTools/src/warptoolConfig.c (revision 29498) @@ -362,4 +362,11 @@ psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -revertcleanup + psMetadata *revertcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + // -donecleanup psMetadata *donecleanupArgs = psMetadataAlloc(); @@ -451,4 +458,5 @@ PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); + PXOPT_ADD_MODE("-revertcleanup", "revert cleanup runs with errors", WARPTOOL_MODE_REVERTCLEANUP, revertcleanupArgs); PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", WARPTOOL_MODE_DONECLEANUP, donecleanupArgs); PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs);