Index: /trunk/ippTasks/Makefile.am =================================================================== --- /trunk/ippTasks/Makefile.am (revision 25418) +++ /trunk/ippTasks/Makefile.am (revision 25419) @@ -17,4 +17,5 @@ warp.pro \ magic.pro \ + destreak.pro \ diff.pro \ stack.pro \ Index: /trunk/ippTasks/destreak.pro =================================================================== --- /trunk/ippTasks/destreak.pro (revision 25419) +++ /trunk/ippTasks/destreak.pro (revision 25419) @@ -0,0 +1,416 @@ +## destreak.pro : support for the streak removal : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/destreak +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToDS +book init magicDSToRevert + +### Database lists +$magicToDS_DB = 0 +$magicDSToRevert_DB = 0 + +#list of stages +$STAGE:n = 0 +list STAGE -add "raw" +list STAGE -add "chip" +list STAGE -add "camera" +list STAGE -add "warp" +list STAGE -add "diff" + +$magicDSStage = 0 +$magicDSRevertStage = 0 + +### Check status of tasks +macro magic.status + book listbook magicToDS + book listbook magicDSToRevert +end + +### Reset tasks +macro magic.reset + book init magicToDS + book init magicDSToRevert +end + +### Turn tasks on +macro destreak.on + task destreak.load + active true + end + task destreak.run + active true + end +end + +macro destreak.revert.on + task destreak.revert.load + active true + end + task destreak.revert.run + active true + end +end +### Turn tasks off +macro destreak.off + task destreak.load + active false + end + task destreak.run + active false + end +end + +macro destreak.revert.off + task destreak.revert.load + active false + end + task destreak.revert.run + active false + end +end + +macro destreak.status + echo magicToDS + book listbook magicToDS + echo "" + echo magicDSToRevert + book listbook magicDSToRevert +end + +task destreak.load + host local + + periods -poll $LOADPOLL + # this query can take a long time (XXX: the long time should be fixed now) + periods -exec 10 + periods -timeout 120 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.destreak.log + + task.exec + $run = magicdstool -todestreak -limit 120 -stage $STAGE:$magicDSStage + $magicDSStage ++ + if ($magicDSStage >= $STAGE:n) set magicDSStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicToDS_DB + $run = $run -dbname $DB:$magicToDS_DB + $magicToDS_DB ++ + if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToDS + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToDS + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicToDS -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicToDS 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicToDS $pageName pantaskState RUN + book getword magicToDS $pageName exp_id -var EXP_ID + book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicToDS $pageName camera -var CAMERA + book getword magicToDS $pageName streaks_uri -var STREAKS + book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS + book getword magicToDS $pageName stage -var STAGE + book getword magicToDS $pageName stage_id -var STAGE_ID + book getword magicToDS $pageName component -var COMPONENT + book getword magicToDS $pageName uri -var URI + book getword magicToDS $pageName path_base -var PATH_BASE + book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE + book getword magicToDS $pageName outroot -var OUTROOT + book getword magicToDS $pageName recoveryroot -var RECROOT + book getword magicToDS $pageName re_place -var REPLACE + book getword magicToDS $pageName dbname -var DBNAME + + sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + end + + # TODO: do not add recoveryroot or replace if they are null or zero + + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicToDS $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicToDS $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicToDS $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicToDS $options:0 pantaskState TIMEOUT + end +end + +task destreak.revert.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.revert.log + + task.exec + $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage + $magicDSRevertStage ++ + if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + + # save the DB name for the exit tasks + option $DB:$magicDSToRevert_DB + $run = $run -dbname $DB:$magicDSToRevert_DB + + # only bump database number after we have gone through all of the stages + if ($magicDSRevertStage == 0) + $magicDSToRevert_DB ++ + end + if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToRevert + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToRevert + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.revert.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages magicDSToRevert -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicDSToRevert $pageName pantaskState RUN + book getword magicDSToRevert $pageName exp_id -var EXP_ID + book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicDSToRevert $pageName camera -var CAMERA + book getword magicDSToRevert $pageName stage -var STAGE + book getword magicDSToRevert $pageName stage_id -var STAGE_ID + book getword magicDSToRevert $pageName component -var COMPONENT + book getword magicDSToRevert $pageName path_base -var PATH_BASE + book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE + book getword magicDSToRevert $pageName outroot -var OUTROOT + book getword magciDSToRevert $pageName bytes -var BYTES + book getword magciDSToRevert $pageName md5sum -var md5sum +# book getword magicDSToRevert $pageName recoveryroot -var RECROOT + book getword magicDSToRevert $pageName re_place -var REPLACE + book getword magicDSToRevert $pageName dbname -var DBNAME + + sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + end + + $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicDSToRevert $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicDSToRevert $options:0 pantaskState TIMEOUT + end +end + +task destreak.completed.revert + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.completed.revert + + task.exec + $run = magicdstool -completedrevert + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicDSToRevert_DB + $run = $run -dbname $DB:$magicDSToRevert_DB + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToRevert + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToRevert + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /trunk/ippTasks/magic.pro =================================================================== --- /trunk/ippTasks/magic.pro (revision 25418) +++ /trunk/ippTasks/magic.pro (revision 25419) @@ -10,21 +10,8 @@ book init magicToTree book init magicToProcess -book init magicToDS -book init magicDSToRevert ### Database lists $magicToTree_DB = 0 $magicToProcess_DB = 0 -$magicToDS_DB = 0 -$magicDSToRevert_DB = 0 - -#list of stages -$STAGE:n = 0 -list STAGE -add "raw" -list STAGE -add "chip" -list STAGE -add "camera" -list STAGE -add "warp" -list STAGE -add "diff" -$magicDSRevertStage = 0 ### Check status of tasks @@ -32,6 +19,4 @@ book listbook magicToTree book listbook magicToProcess - book listbook magicToDS - book listbook magicDSToRevert end @@ -40,6 +25,4 @@ book init magicToTree book init magicToProcess - book init magicToDS - book init magicDSToRevert end @@ -58,20 +41,6 @@ active true end - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.revert.on - task magic.ds.revert.load - active true - end - task magic.ds.revert.run - active true - end -end +end + ### Turn tasks off macro magic.off @@ -88,46 +57,5 @@ active false end - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end - -macro magic.ds.revert.off - task magic.ds.revert.load - active false - end - task magic.ds.revert.run - active false - end -end -macro magic.ds.off - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end -macro magic.ds.on - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.status - echo magicToDS - book listbook magicToDS - echo "" - echo magicDSToRevert - book listbook magicDSToRevert -end - - +end task magic.tree.load @@ -400,279 +328,2 @@ end end - -task magic.destreak.load - host local - - periods -poll $LOADPOLL - # this query can take a long time - periods -exec 10 - periods -timeout 120 - npending 1 - - stdout NULL - stderr $LOGDIR/magic.destreak.log - - task.exec - $run = magicdstool -todestreak -limit 120 - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$magicToDS_DB - $run = $run -dbname $DB:$magicToDS_DB - $magicToDS_DB ++ - if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicToDS - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicToDS - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task magic.destreak.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - - task.exec - book npages magicToDS -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicToDS 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicToDS $pageName pantaskState RUN - book getword magicToDS $pageName exp_id -var EXP_ID - book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicToDS $pageName camera -var CAMERA - book getword magicToDS $pageName streaks_uri -var STREAKS - book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS - book getword magicToDS $pageName stage -var STAGE - book getword magicToDS $pageName stage_id -var STAGE_ID - book getword magicToDS $pageName component -var COMPONENT - book getword magicToDS $pageName uri -var URI - book getword magicToDS $pageName path_base -var PATH_BASE - book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE - book getword magicToDS $pageName outroot -var OUTROOT - book getword magicToDS $pageName recoveryroot -var RECROOT - book getword magicToDS $pageName re_place -var REPLACE - book getword magicToDS $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - # TODO: do not add recoveryroot or replace if they are null or zero - - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicToDS $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicToDS $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicToDS $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicToDS $options:0 pantaskState TIMEOUT - end -end - -task magic.ds.revert.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 20 - npending 1 - active false - - stdout NULL - stderr $LOGDIR/magic.ds.revert.log - - task.exec - $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage - $magicDSRevertStage ++ - if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0 - - if ($DB:n == 0) - option DEFAULT - else - - # save the DB name for the exit tasks - option $DB:$magicDSToRevert_DB - $run = $run -dbname $DB:$magicDSToRevert_DB - - # only bump database number after we have gone through all of the stages - if ($magicDSRevertStage == 0) - $magicDSToRevert_DB ++ - end - if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicDSToRevert - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicDSToRevert - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task magic.ds.revert.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active false - - task.exec - book npages magicDSToRevert -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicDSToRevert $pageName pantaskState RUN - book getword magicDSToRevert $pageName exp_id -var EXP_ID - book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicDSToRevert $pageName camera -var CAMERA - book getword magicDSToRevert $pageName stage -var STAGE - book getword magicDSToRevert $pageName stage_id -var STAGE_ID - book getword magicDSToRevert $pageName component -var COMPONENT - book getword magicDSToRevert $pageName path_base -var PATH_BASE - book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE - book getword magicDSToRevert $pageName outroot -var OUTROOT - book getword magciDSToRevert $pageName bytes -var BYTES - book getword magciDSToRevert $pageName md5sum -var md5sum -# book getword magicDSToRevert $pageName recoveryroot -var RECROOT - book getword magicDSToRevert $pageName re_place -var REPLACE - book getword magicDSToRevert $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicDSToRevert $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicDSToRevert $options:0 pantaskState TIMEOUT - end -end - Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 25418) +++ /trunk/ippTools/share/Makefile.am (revision 25419) @@ -165,4 +165,5 @@ magictool_revertnode.sql \ magicdstool_completed_runs.sql \ + magicdstool_completedrevert.sql \ magicdstool_definebyquery_raw.sql \ magicdstool_definebyquery_chip.sql \ @@ -172,5 +173,9 @@ magicdstool_getrunids.sql \ magicdstool_getskycells.sql \ - magicdstool_todestreak.sql \ + magicdstool_todestreak_camera.sql \ + magicdstool_todestreak_chip.sql \ + magicdstool_todestreak_diff.sql \ + magicdstool_todestreak_raw.sql \ + magicdstool_todestreak_warp.sql \ magicdstool_toremove.sql \ magicdstool_torestore.sql \ Index: /trunk/ippTools/share/magicdstool_completedrevert.sql =================================================================== --- /trunk/ippTools/share/magicdstool_completedrevert.sql (revision 25419) +++ /trunk/ippTools/share/magicdstool_completedrevert.sql (revision 25419) @@ -0,0 +1,10 @@ +SELECT + magic_ds_id, + state +FROM magicDSRun +LEFT JOIN magicDSFile USING(magic_ds_id) +WHERE + magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored' +GROUP BY magic_ds_id +HAVING COUNT(component) = 0 + Index: unk/ippTools/share/magicdstool_todestreak.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak.sql (revision 25418) +++ (revision ) @@ -1,228 +1,0 @@ -SELECT * -FROM ( -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - class_id as component, - rawImfile.uri as uri, - NULL as path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING (magic_id) -JOIN camProcessedExp USING(cam_id) -JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = rawImfile.class_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'raw' - AND magicDSFile.component IS NULL -UNION - -- chipProcesedImfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicDSRun.magic_id, - chipRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - class_id as component, - chipProcessedImfile.uri, - chipProcessedImfile.path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING(magic_id) -JOIN camProcessedExp USING(cam_id) -JOIN chipRun ON chip_id = stage_id -JOIN chipProcessedImfile USING(chip_id) -JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = chipProcessedImfile.class_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'chip' - AND chipRun.state = 'full' - AND chipProcessedImfile.fault = 0 - AND chipProcessedImfile.quality = 0 - AND magicDSFile.component IS NULL -UNION - -- camProcessedExp -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicDSRun.magic_id, - chipRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - 'exposure' as component, - NULL AS uri, - camProcessedExp.path_base, - magicRun.inverse, - camProcessedExp.path_base as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING(magic_id) -JOIN camRun ON magicDSRun.stage_id = camRun.cam_id -JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id -JOIN chipRun USING(chip_id) -JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'camera' - AND camRun.state = 'full' - AND chipRun.state = 'full' - AND chipRun.magicked - AND camProcessedExp.fault = 0 - AND camProcessedExp.quality = 0 - AND magicDSFile.component IS NULL -UNION --- warpSkyfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - stage_id, - warpSkyfile.skycell_id as component, - warpSkyfile.uri, - warpSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM magicDSRun -JOIN magicMask USING (magic_id) -JOIN magicRun USING (magic_id) -JOIN warpRun ON warp_id = stage_id -JOIN warpSkyfile USING(warp_id) -JOIN rawExp ON magicRun.exp_id = rawExp.exp_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = warpSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'warp' - AND warpRun.state = 'full' - AND warpSkyfile.fault = 0 - AND warpSkyfile.quality = 0 - AND magicDSFile.component IS NULL -UNION --- regular diffSkyfiles -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - rawExp.camera, - magicMask.uri as streaks_uri, - NULL AS inv_streaks_uri, - stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, - NULL AS uri, - diffSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM rawExp -JOIN magicRun USING (exp_id) -JOIN magicMask USING (magic_id) -JOIN magicDSRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile - ON magicRun.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'diff' - AND diffRun.bothways = 0 - AND diffSkyfile.fault = 0 - AND diffSkyfile.quality = 0 - AND magicDSFile.component IS NULL --- bothways diffSkyfiles -UNION -SELECT DISTINCT - magicDSRun.magic_ds_id, - magicRun.magic_id, - magicRun.exp_id, - magicDSRun.label, - rawExp.camera, - magicMask.uri as streaks_uri, - (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, - stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, - NULL AS uri, - diffSkyfile.path_base, - magicRun.inverse, - NULL as cam_path_base, - outroot, - recoveryroot, - re_place, - remove -FROM rawExp -JOIN magicRun USING (exp_id) -JOIN magicMask USING (magic_id) -JOIN magicDSRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id) -JOIN diffRun USING(diff_id) -JOIN diffSkyfile - ON magicRun.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -LEFT JOIN magicDSFile - ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND magicDSFile.component = diffSkyfile.skycell_id -WHERE - magicDSRun.state = 'new' - AND magicDSRun.stage = 'diff' - AND diffRun.bothways - AND diffSkyfile.fault = 0 - AND diffSkyfile.quality = 0 - AND magicDSFile.component IS NULL -) as Foo Index: /trunk/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_camera.sql (revision 25419) +++ /trunk/ippTools/share/magicdstool_todestreak_camera.sql (revision 25419) @@ -0,0 +1,37 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + chipRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + 'exposure' as component, + CAST(NULL AS CHAR(255)) AS uri, + camProcessedExp.path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camRun ON magicDSRun.stage_id = camRun.cam_id +JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id +JOIN chipRun USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'camera' + AND camRun.state = 'full' + AND chipRun.state = 'full' + AND chipRun.magicked + AND camProcessedExp.fault = 0 + AND camProcessedExp.quality = 0 + AND magicDSFile.component IS NULL Index: /trunk/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_chip.sql (revision 25419) +++ /trunk/ippTools/share/magicdstool_todestreak_chip.sql (revision 25419) @@ -0,0 +1,36 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicDSRun.magic_id, + chipRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + class_id as component, + chipProcessedImfile.uri, + chipProcessedImfile.path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun ON chip_id = stage_id +JOIN chipProcessedImfile USING(chip_id) +JOIN rawExp ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'chip' + AND chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 + AND magicDSFile.component IS NULL Index: /trunk/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_diff.sql (revision 25419) +++ /trunk/ippTools/share/magicdstool_todestreak_diff.sql (revision 25419) @@ -0,0 +1,82 @@ +SELECT * FROM ( +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways = 0 + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL +-- bothways diffSkyfiles +UNION +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + CAST(NULL AS CHAR(255)) AS uri, + diffSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL +) as magicDSRun +-- we need the following so this query is compatible with the other stages +WHERE magic_ds_id IS NOT NULL Index: /trunk/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_raw.sql (revision 25419) +++ /trunk/ippTools/share/magicdstool_todestreak_raw.sql (revision 25419) @@ -0,0 +1,31 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + class_id as component, + rawImfile.uri as uri, + CAST(NULL AS CHAR(255)) as path_base, + magicRun.inverse, + camProcessedExp.path_base as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN camProcessedExp USING(cam_id) +JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = rawImfile.class_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'raw' + AND magicDSFile.component IS NULL Index: /trunk/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_warp.sql (revision 25419) +++ /trunk/ippTools/share/magicdstool_todestreak_warp.sql (revision 25419) @@ -0,0 +1,35 @@ +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + camera, + magicMask.uri as streaks_uri, + CAST(NULL AS CHAR(255)) AS inv_streaks_uri, + stage, + stage_id, + warpSkyfile.skycell_id as component, + warpSkyfile.uri, + warpSkyfile.path_base, + magicRun.inverse, + CAST(NULL AS CHAR(255)) as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM magicDSRun +JOIN magicMask USING (magic_id) +JOIN magicRun USING (magic_id) +JOIN warpRun ON warp_id = stage_id +JOIN warpSkyfile USING(warp_id) +JOIN rawExp ON magicRun.exp_id = rawExp.exp_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = warpSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'warp' + AND warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND magicDSFile.component IS NULL Index: /trunk/ippTools/share/magicdstool_torevert_camera.sql =================================================================== --- /trunk/ippTools/share/magicdstool_torevert_camera.sql (revision 25418) +++ /trunk/ippTools/share/magicdstool_torevert_camera.sql (revision 25419) @@ -19,4 +19,5 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'camera' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: /trunk/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- /trunk/ippTools/share/magicdstool_torevert_chip.sql (revision 25418) +++ /trunk/ippTools/share/magicdstool_torevert_chip.sql (revision 25419) @@ -18,4 +18,5 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'chip' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: /trunk/ippTools/share/magicdstool_torevert_diff.sql =================================================================== --- /trunk/ippTools/share/magicdstool_torevert_diff.sql (revision 25418) +++ /trunk/ippTools/share/magicdstool_torevert_diff.sql (revision 25419) @@ -18,4 +18,5 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'diff' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: /trunk/ippTools/share/magicdstool_torevert_raw.sql =================================================================== --- /trunk/ippTools/share/magicdstool_torevert_raw.sql (revision 25418) +++ /trunk/ippTools/share/magicdstool_torevert_raw.sql (revision 25419) @@ -18,4 +18,5 @@ AND rawImfile.tmp_class_id = summitImfile.class_id) WHERE magicDSRun.stage = 'raw' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: /trunk/ippTools/share/magicdstool_torevert_warp.sql =================================================================== --- /trunk/ippTools/share/magicdstool_torevert_warp.sql (revision 25418) +++ /trunk/ippTools/share/magicdstool_torevert_warp.sql (revision 25419) @@ -21,4 +21,5 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'warp' - AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') - AND magicDSFile.fault > 0 + AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + OR magicDSRun.state = 'goto_censored' + OR magicDSRun.state = 'goto_restored') Index: /trunk/ippTools/share/magictool_censor_chip.sql =================================================================== --- /trunk/ippTools/share/magictool_censor_chip.sql (revision 25418) +++ /trunk/ippTools/share/magictool_censor_chip.sql (revision 25419) @@ -1,7 +1,7 @@ UPDATE chipRun JOIN chipProcessedImfile USING(chip_id, exp_id) - JOIN magicDSRun ON chipRun.magicked = magicDSRun.magic_ds_id + JOIN magicDSRun ON stage_id = chip_id AND stage = 'chip' AND re_place JOIN magicDSFile USING(magic_ds_id) - JOIN magicRun USING(magic_id, exp_id) + JOIN magicRun using(magic_id) SET chipProcessedImfile.magicked = 0, chipRun.magicked = 0, Index: /trunk/ippTools/share/magictool_censor_diff.sql =================================================================== --- /trunk/ippTools/share/magictool_censor_diff.sql (revision 25418) +++ /trunk/ippTools/share/magictool_censor_diff.sql (revision 25419) @@ -1,7 +1,7 @@ -UPDATE magicRun - JOIN magicDSRun USING(magic_id) +UPDATE diffRun + JOIN diffSkyfile USING(diff_id) + JOIN magicRun USING(diff_id) + JOIN magicDSRun ON stage = 'diff' AND stage_id = diff_id AND re_place JOIN magicDSFile USING(magic_ds_id) - JOIN diffRun ON magic_ds_id = magicked - JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id SET diffRun.magicked = 0, diffSkyfile.magicked = 0, Index: /trunk/ippTools/share/magictool_censor_raw.sql =================================================================== --- /trunk/ippTools/share/magictool_censor_raw.sql (revision 25418) +++ /trunk/ippTools/share/magictool_censor_raw.sql (revision 25419) @@ -2,9 +2,9 @@ JOIN magicDSRun USING(magic_id) JOIN magicDSFile USING(magic_ds_id) - JOIN rawExp ON rawExp.magicked = magic_ds_id - JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id + JOIN rawImfile ON rawImfile.magicked = magic_ds_id +-- JOIN rawExp ON rawExp.magicked = magic_ds_id SET rawImfile.magicked = 0, - rawExp.magicked = 0, - magicDSRun.state = 'censored', +-- rawExp.magicked = 0, +-- magicDSRun.state = 'censored', magicDSFile.fault = 42 Index: /trunk/ippTools/src/magicdstool.c =================================================================== --- /trunk/ippTools/src/magicdstool.c (revision 25418) +++ /trunk/ippTools/src/magicdstool.c (revision 25419) @@ -42,4 +42,5 @@ static bool torestoreMode(pxConfig *config); static bool torevertMode(pxConfig *config); +static bool completedrevertMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -75,4 +76,5 @@ MODECASE(MAGICDSTOOL_MODE_TORESTORE, torestoreMode); MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); + MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); default: psAbort("invalid option (this should not happen)"); @@ -396,23 +398,28 @@ PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_todestreak.sql"); + psString sql_file = NULL; + psStringAppend(&sql_file, "magicdstool_todestreak_%s.sql", stage); + + psString query = pxDataGet(sql_file); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psFree(sql_file); + return false; + } + psFree(sql_file); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -817,36 +824,37 @@ return true; } - -static bool getskycellsMode(pxConfig *config) -{ - // required - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); +static bool completedrevertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magicdstool_getskycells.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("magicdstool_completedrevert.sql"); + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + psString whereString = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereString, "\nAND %s", whereClause); psFree(whereClause); } psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) { + psFree(whereString); + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + psFree(whereString); psFree(query); - psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -868,14 +876,24 @@ return true; } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); + for (int i=0; idata[i]; + psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); + psString old_state = psMetadataLookupStr(NULL, row, "state"); + psString new_state; + if (!strcmp(old_state, "goto_censored")) { + new_state = "censored"; + } else if (!strcmp(old_state, "goto_restored")) { + new_state = "restored"; + } else { + psError(PXTOOLS_ERR_PROG, true, "unexpected state found: %s", old_state); psFree(output); return false; } - } - + char *query2 = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query2, magic_ds_id, new_state)) { + psError(PS_ERR_UNKNOWN, false, "failed to set run magicDSRun.state to %s", new_state); + return false; + } + } psFree(output); @@ -883,43 +901,16 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!( - (strncmp(state, "new", 4) == 0) - || (strncmp(state, "full", 5) == 0) - ) - ) { - psError(PS_ERR_UNKNOWN, false, - "invalid magicDSRun state: %s", state); - return false; - } - - char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for magic_id %" PRId64, magic_ds_id); - return false; - } - - return true; -} - -static bool toremoveMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); +static bool getskycellsMode(pxConfig *config) +{ + // required + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_toremove.sql"); + psString query = pxDataGet("magicdstool_getskycells.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -929,17 +920,10 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } psFree(where); - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -970,5 +954,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -981,5 +965,31 @@ return true; } -static bool torestoreMode(pxConfig *config) + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "new", 4) == 0) + || (strncmp(state, "full", 5) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid magicDSRun state: %s", state); + return false; + } + + char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for magic_id %" PRId64, magic_ds_id); + return false; + } + + return true; +} + +static bool toremoveMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -994,5 +1004,5 @@ // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_torestore.sql"); + psString query = pxDataGet("magicdstool_toremove.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -1043,5 +1053,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -1054,34 +1064,26 @@ return true; } - - -static bool torevertMode(pxConfig *config) +static bool torestoreMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString sql_file = NULL; - psStringAppend(&sql_file, "magicdstool_torevert_%s.sql", stage); - - psString query = pxDataGet(sql_file); + // look for "inputs" that need to processed + psString query = pxDataGet("magicdstool_torestore.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); - psFree(sql_file); - return false; - } - psFree(sql_file); + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); } @@ -1124,4 +1126,85 @@ if (psArrayLength(output)) { // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool torevertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString sql_file = NULL; + psStringAppend(&sql_file, "magicdstool_torevert_%s.sql", stage); + + psString query = pxDataGet(sql_file); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psFree(sql_file); + return false; + } + psFree(sql_file); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true if (!ippdbPrintMetadatas(stdout, output, "torevert", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); Index: /trunk/ippTools/src/magicdstool.h =================================================================== --- /trunk/ippTools/src/magicdstool.h (revision 25418) +++ /trunk/ippTools/src/magicdstool.h (revision 25419) @@ -35,4 +35,5 @@ MAGICDSTOOL_MODE_TORESTORE, MAGICDSTOOL_MODE_TOREVERT, + MAGICDSTOOL_MODE_COMPLETEDREVERT, } MAGICDStoolMode; Index: /trunk/ippTools/src/magicdstoolConfig.c =================================================================== --- /trunk/ippTools/src/magicdstoolConfig.c (revision 25418) +++ /trunk/ippTools/src/magicdstoolConfig.c (revision 25419) @@ -93,8 +93,8 @@ // -todestreak psMetadata *todestreakArgs = psMetadataAlloc(); + psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage (required)", NULL); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL); psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -147,4 +147,10 @@ psMetadataAddU64(torevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -completedrevert + psMetadata *completedrevertArgs = psMetadataAlloc(); + psMetadataAddS64(completedrevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(completedrevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(completedrevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psFree(now); @@ -174,4 +180,7 @@ MAGICDSTOOL_MODE_TOREVERT, torevertArgs); + PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting", + MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { psError(PS_ERR_UNKNOWN, true, "option parsing failed");