Index: /branches/eam_branches/20090715/ippTools/doc/addstar_flow.txt =================================================================== --- /branches/eam_branches/20090715/ippTools/doc/addstar_flow.txt (revision 25400) +++ /branches/eam_branches/20090715/ippTools/doc/addstar_flow.txt (revision 25400) @@ -0,0 +1,34 @@ +### Set up the addRun row during the previous stage +#>> camtool.c @ addprocessedexpMode(pxConfig *config) +camtool -cam_id $cam_id -addprocessedexp ... + +### Catch that things now need to be added. This should check for magic_status as well, probably. +#>> add.pro / addtool.c / addtoolConfig.c +addtool -pendingsmf +#addMe METADATA +# add_id S64 5 +#END + +### Run the addstar script. +#>> addstar_run.pl +addstar_run.pl --add_id $add_id --camera $camera --camroot=$camroot --dvodb $DVODB + addtool -pendingsmf -add_id $add_id +#addMe METADATA +# add_id S64 5 +# workdir STR /path/to/workdir +# +#END + + ppConfigDump -camera $camera -recipe ADDSTAR $recipt_addstar -dump-recipe ADDSTAR - + addstar -D CAMERA $camdir -update -D CATDIR $dvodbReal $realFile + addtool -add_id $add_id -addprocessedsmf ... + addtool -add_id $add_id -updaterun -state full + +### Reverting +#>> addtool.c +addtool -revertprocessedsmf ... + + + + + Index: /branches/eam_branches/20090715/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/20090715/ippTools/share/Makefile.am (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/Makefile.am (revision 25400) @@ -4,4 +4,8 @@ dist_pkgdata_DATA = \ + addtool_find_pendingexp.sql \ + addtool_queue_cam_id.sql \ + addtool_reset_faulted_runs.sql \ + addtool_revertprocessedexp.sql \ camtool_donecleanup.sql \ camtool_find_chip_id.sql \ @@ -43,4 +47,5 @@ dettool_normalizedstat.sql \ dettool_pending.sql \ + dettool_pendingcleanup_detrunsummary.sql \ dettool_pendingcleanup_normalizedexp.sql \ dettool_pendingcleanup_normalizedimfile.sql \ @@ -175,10 +180,12 @@ magicdstool_torevert_warp.sql \ magicdstool_torevert_diff.sql \ - pstamptool_addjob_otherjob.sql \ - pstamptool_addjob_stampjob.sql \ + pstamptool_completedreq.sql \ pstamptool_datastore.sql \ pstamptool_pendingjob.sql \ pstamptool_pendingreq.sql \ pstamptool_project.sql \ + pstamptool_revertjob.sql \ + pstamptool_revertreq.sql \ + pstamptool_revertreq_deletejobs.sql \ pxadmin_create_tables.sql \ pxadmin_create_mirror_tables.sql \ Index: /branches/eam_branches/20090715/ippTools/share/addtool_find_pendingexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/addtool_find_pendingexp.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/addtool_find_pendingexp.sql (revision 25400) @@ -0,0 +1,26 @@ +SELECT + addRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM addRun +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN addProcessedExp + USING(add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + camRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addMask.label IS NULL + Index: /branches/eam_branches/20090715/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/addtool_queue_cam_id.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/addtool_queue_cam_id.sql (revision 25400) @@ -0,0 +1,14 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + cam_id, -- cam_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- label + '%s', -- dvodb + 0 -- magicked + FROM camRun + WHERE + camRun.state = 'full' + AND camRun.cam_id = %lld Index: /branches/eam_branches/20090715/ippTools/share/addtool_reset_faulted_runs.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/addtool_reset_faulted_runs.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/addtool_reset_faulted_runs.sql (revision 25400) @@ -0,0 +1,8 @@ +UPDATE addRun, addProcessedExp, camRun, chipRun, rawExp +SET addRun.state = 'new' +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 Index: /branches/eam_branches/20090715/ippTools/share/addtool_revertprocessedexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/addtool_revertprocessedexp.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/addtool_revertprocessedexp.sql (revision 25400) @@ -0,0 +1,8 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun, camRun, chipRun, rawExp +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 Index: /branches/eam_branches/20090715/ippTools/share/camtool_pendingcleanupexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/camtool_pendingcleanupexp.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/camtool_pendingcleanupexp.sql (revision 25400) @@ -13,4 +13,4 @@ USING(cam_id) WHERE - (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged') + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged' OR camRun.state = 'goto_scrubbed') Index: /branches/eam_branches/20090715/ippTools/share/camtool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/camtool_pendingcleanuprun.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/camtool_pendingcleanuprun.sql (revision 25400) @@ -9,3 +9,3 @@ USING (exp_id) WHERE - (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged') + (camRun.state = 'goto_cleaned' OR camRun.state = 'goto_purged' OR camRun.state = 'goto_scrubbed') Index: /branches/eam_branches/20090715/ippTools/share/chiptool_processedimfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/chiptool_processedimfile.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/chiptool_processedimfile.sql (revision 25400) @@ -9,4 +9,5 @@ chipProcessedImfile.path_base, chipProcessedImfile.magicked, + chipProcessedImfile.data_state, chipRun.state, chipRun.workdir, Index: /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedexp.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedexp.sql (revision 25400) @@ -0,0 +1,11 @@ +SELECT + detNormalizedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedExp + USING(det_id,iteration) +WHERE + (detNormalizedExp.data_state = 'cleaned' + OR detNormalizedExp.data_state = 'scrubbed' + OR detNormalizedExp.data_state = 'purged') + Index: /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedimfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedimfile.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedimfile.sql (revision 25400) @@ -0,0 +1,11 @@ +SELECT + detNormalizedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedImfile + USING(det_id,iteration) +WHERE + (detNormalizedImfile.data_state = 'cleaned' + OR detNormalizedImfile.data_state = 'scrubbed' + OR detNormalizedImfile.data_state = 'purged') + Index: /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedstat.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedstat.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_normalizedstat.sql (revision 25400) @@ -0,0 +1,11 @@ +SELECT + detNormalizedStatImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detNormalizedStatImfile + USING(det_id,iteration) +WHERE + (detNormalizedStatImfile.data_state = 'cleaned' + OR detNormalizedStatImfile.data_state = 'scrubbed' + OR detNormalizedStatImfile.data_state = 'purged') + Index: /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_processedexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_processedexp.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_processedexp.sql (revision 25400) @@ -0,0 +1,11 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedExp + USING(det_id) +WHERE + (detProcessedExp.data_state = 'cleaned' + OR detProcessedExp.data_state = 'scrubbed' + OR detProcessedExp.data_state = 'purged') Index: /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_processedimfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_processedimfile.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_processedimfile.sql (revision 25400) @@ -0,0 +1,11 @@ +-- need to restrict to a single detRunSummary (require all to say 'cleaned'?) +SELECT + detProcessedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detProcessedImfile + USING(det_id) +WHERE + (detProcessedImfile.data_state = 'cleaned' + OR detProcessedImfile.data_state = 'scrubbed' + OR detProcessedImfile.data_state = 'purged') Index: /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_residexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_residexp.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_residexp.sql (revision 25400) @@ -0,0 +1,10 @@ +SELECT + detResidExp.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidExp + USING(det_id,iteration) +WHERE + (detResidExp.data_state = 'cleaned' + OR detResidExp.data_state = 'scrubbed' + OR detResidExp.data_state = 'purged') Index: /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_residimfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_residimfile.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_residimfile.sql (revision 25400) @@ -0,0 +1,12 @@ +SELECT + detResidImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detResidImfile + USING(det_id,iteration) +WHERE + (detResidImfile.data_state = 'cleaned' + OR detResidImfile.data_state = 'scrubbed' + OR detResidImfile.data_state = 'purged') + + Index: /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_stacked.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_stacked.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_donecleanup_stacked.sql (revision 25400) @@ -0,0 +1,10 @@ +SELECT + detStackedImfile.*, + detRunSummary.data_state +FROM detRunSummary +JOIN detStackedImfile + USING(det_id,iteration) +WHERE + (detStackedImfile.data_state = 'cleaned' + OR detStackedImfile.data_state = 'scrubbed' + OR detStackedImfile.data_state = 'purged') Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_detrunsummary.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_detrunsummary.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_detrunsummary.sql (revision 25400) @@ -0,0 +1,42 @@ +SELECT DISTINCT detRunSummary.* +FROM detRunSummary + JOIN detProcessedImfile USING (det_id) + JOIN detProcessedExp USING (det_id) + JOIN detStackedImfile USING (det_id,iteration) + JOIN detResidExp USING (det_id,iteration) + JOIN detResidImfile USING (det_id,iteration) + JOIN detNormalizedStatImfile USING (det_id,iteration) + JOIN detNormalizedImfile USING (det_id,iteration) + JOIN detNormalizedExp USING (det_id,iteration) +WHERE (!( + ( + detRunSummary.data_state = 'full' OR + detRunSummary.data_state = 'cleaned' + ) OR + ( + !((detProcessedImfile.data_state = 'cleaned' OR + detProcessedImfile.data_state = 'scrubbed' OR + detProcessedImfile.data_state = 'purged') AND + (detProcessedExp.data_state = 'cleaned' OR + detProcessedExp.data_state = 'scrubbed' OR + detProcessedExp.data_state = 'purged') AND + (detStackedImfile.data_state = 'cleaned' OR + detStackedImfile.data_state = 'scrubbed' OR + detStackedImfile.data_state = 'purged') AND + (detResidExp.data_state = 'cleaned' OR + detResidExp.data_state = 'scrubbed' OR + detResidExp.data_state = 'purged') AND + (detResidImfile.data_state = 'cleaned' OR + detResidImfile.data_state = 'scrubbed' OR + detResidImfile.data_state = 'purged') AND + (detNormalizedStatImfile.data_state = 'cleaned' OR + detNormalizedStatImfile.data_state = 'scrubbed' OR + detNormalizedStatImfile.data_state = 'purged') AND + (detNormalizedImfile.data_state = 'cleaned' OR + detNormalizedImfile.data_state = 'scrubbed' OR + detNormalizedImfile.data_state = 'purged') AND + (detNormalizedExp.data_state = 'cleaned' OR + detNormalizedExp.data_state = 'scrubbed' OR + detNormalizedExp.data_state = 'purged') + ) + ))) Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedexp.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedexp.sql (revision 25400) @@ -1,9 +1,21 @@ SELECT detNormalizedExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedExp USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedExp.data_state = 'full') + OR detNormalizedExp.data_state = 'goto_cleaned' + OR detNormalizedExp.data_state = 'goto_scrubbed' + OR detNormalizedExp.data_state = 'goto_purged') + Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 25400) @@ -1,9 +1,21 @@ -SELECT +SELECT DISTINCT detNormalizedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedImfile.data_state = 'full') + OR detNormalizedImfile.data_state = 'goto_cleaned' + OR detNormalizedImfile.data_state = 'goto_scrubbed' + OR detNormalizedImfile.data_state = 'goto_purged') + Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedstat.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedstat.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_normalizedstat.sql (revision 25400) @@ -1,9 +1,21 @@ -SELECT +SELECT DISTINCT detNormalizedStatImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detNormalizedStatImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detNormalizedStatImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detNormalizedStatImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detNormalizedStatImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detNormalizedStatImfile.data_state = 'full') + OR detNormalizedStatImfile.data_state = 'goto_cleaned' + OR detNormalizedStatImfile.data_state = 'goto_scrubbed' + OR detNormalizedStatImfile.data_state = 'goto_purged') + Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_processedexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_processedexp.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_processedexp.sql (revision 25400) @@ -2,9 +2,18 @@ SELECT detProcessedExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detProcessedExp USING(det_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detProcessedExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detProcessedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detProcessedExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detProcessedExp.data_state = 'full') + OR detProcessedExp.data_state = 'goto_cleaned' + OR detProcessedExp.data_state = 'goto_scrubbed' + OR detProcessedExp.data_state = 'goto_purged') Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_processedimfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_processedimfile.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_processedimfile.sql (revision 25400) @@ -1,10 +1,15 @@ --- need to restrict to a single detRunSummary (require all to say 'cleaned'?) SELECT detProcessedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detProcessedImfile USING(det_id) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detProcessedImfile.data_state = 'full' + (detProcessedImfile.data_state = 'goto_cleaned' + OR detProcessedImfile.data_state = 'goto_scrubbed' + OR detProcessedImfile.data_state = 'goto_purged') + Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_residexp.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_residexp.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_residexp.sql (revision 25400) @@ -1,9 +1,20 @@ SELECT detResidExp.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detResidExp USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detResidExp.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detResidExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detResidExp.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detResidExp.data_state = 'full') + OR detResidExp.data_state = 'goto_cleaned' + OR detResidExp.data_state = 'goto_scrubbed' + OR detResidExp.data_state = 'goto_purged') Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_residimfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_residimfile.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_residimfile.sql (revision 25400) @@ -1,9 +1,22 @@ SELECT detResidImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detResidImfile USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration,exp_id) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detResidImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detResidImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detResidImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detResidImfile.data_state = 'full') + OR detResidImfile.data_state = 'goto_cleaned' + OR detResidImfile.data_state = 'goto_scrubbed' + OR detResidImfile.data_state = 'goto_purged') + + Index: /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_stacked.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 25400) @@ -1,9 +1,20 @@ -SELECT +SELECT DISTINCT detStackedImfile.*, - detRunSummary.data_state + rawExp.camera FROM detRunSummary JOIN detStackedImfile - USING(det_id,iteration) + USING(det_id,iteration) +JOIN detInputExp + USING(det_id,iteration) +JOIN rawExp + USING(exp_id) WHERE - detRunSummary.data_state = 'goto_cleaned' -AND detStackedImfile.data_state = 'full' + ((detRunSummary.data_state = 'goto_cleaned' + AND detStackedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_scrubbed' + AND detStackedImfile.data_state = 'full') + OR (detRunSummary.data_state = 'goto_purged' + AND detStackedImfile.data_state = 'full') + OR detStackedImfile.data_state = 'goto_cleaned' + OR detStackedImfile.data_state = 'goto_scrubbed' + OR detStackedImfile.data_state = 'goto_purged') Index: /branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part1.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part1.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/difftool_definewarpstack_part1.sql (revision 25400) @@ -15,4 +15,5 @@ SELECT DISTINCT diffRun.*, + warp_id, exp_id FROM diffRun @@ -24,5 +25,5 @@ JOIN chipRun USING(chip_id) WHERE warp1 IS NOT NULL -) AS diffExp USING(exp_id) +) AS diffExp USING(exp_id, warp_id) WHERE warpSkyfile.fault = 0 Index: /branches/eam_branches/20090715/ippTools/share/difftool_skyfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/difftool_skyfile.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/difftool_skyfile.sql (revision 25400) @@ -6,25 +6,56 @@ diffRun.workdir, diffRun.bothways, - camProcessedExp.zpt_obs, - camProcessedExp.zpt_stdev, - camProcessedExp.zpt_lq, - camProcessedExp.zpt_uq, - rawExp.exp_time, - rawExp.camera, warp1, stack1, warp2, - stack2 + stack2, + -- The following are only valid for warps + -- XXX This needs to be more clever to handle diffs between stacks + -- Zero points are appropriate for both forward and backward diffs + camProcessedInput.zpt_obs, + camProcessedInput.zpt_stdev, + camProcessedInput.zpt_lq, + camProcessedInput.zpt_uq, + rawInput.exp_time, + rawInput.camera, + rawInput.exp_name AS exp_name_1, + rawInput.exp_id AS exp_id_1, + chipInput.chip_id AS chip_id_1, + camInput.cam_id AS cam_id_1, + fakeInput.fake_id AS fake_id_1, + camProcessedInput.sigma_ra AS sigma_ra_1, + camProcessedInput.sigma_dec AS sigma_dec_1, + rawTemplate.exp_name AS exp_name_2, + rawTemplate.exp_id AS exp_id_2, + chipTemplate.chip_id AS chip_id_2, + camTemplate.cam_id AS cam_id_2, + fakeTemplate.fake_id AS fake_id_2, + camProcessedTemplate.sigma_ra AS sigma_ra_2, + camProcessedTemplate.sigma_dec AS sigma_dec_2 FROM diffRun JOIN diffSkyfile USING(diff_id) JOIN diffInputSkyfile USING(diff_id, skycell_id) -JOIN warpRun --- NOTE: joining input only! --- This is so that we can get the correct zero point --- XXX This needs to be more clever to handle diffs between stacks - ON warpRun.warp_id = diffInputSkyfile.warp1 -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN camProcessedExp USING(cam_id) -JOIN chipRun USING(chip_id) -JOIN rawExp USING(exp_id) +LEFT JOIN warpRun AS warpInput + ON warpInput.warp_id = diffInputSkyfile.warp1 +LEFT JOIN fakeRun AS fakeInput + ON fakeInput.fake_id = warpInput.fake_id +LEFT JOIN camRun AS camInput + ON camInput.cam_id = fakeInput.cam_id +LEFT JOIN camProcessedExp AS camProcessedInput + ON camProcessedInput.cam_id = camInput.cam_id +LEFT JOIN chipRun AS chipInput + ON chipInput.chip_id = camInput.chip_id +LEFT JOIN rawExp AS rawInput + ON rawInput.exp_id = chipInput.exp_id +LEFT JOIN warpRun AS warpTemplate + ON warpTemplate.warp_id = diffInputSkyfile.warp2 +LEFT JOIN fakeRun AS fakeTemplate + ON fakeTemplate.fake_id = warpTemplate.fake_id +LEFT JOIN camRun AS camTemplate + ON camTemplate.cam_id = fakeTemplate.cam_id +LEFT JOIN camProcessedExp AS camProcessedTemplate + ON camProcessedTemplate.cam_id = camTemplate.cam_id +LEFT JOIN chipRun AS chipTemplate + ON chipTemplate.chip_id = camTemplate.chip_id +LEFT JOIN rawExp AS rawTemplate + ON rawTemplate.exp_id = chipTemplate.exp_id Index: /branches/eam_branches/20090715/ippTools/share/faketool_pendingcleanupimfile.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/faketool_pendingcleanupimfile.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/faketool_pendingcleanupimfile.sql (revision 25400) @@ -13,3 +13,7 @@ USING(fake_id) WHERE - fakeRun.state = 'goto_cleaned' + ((fakeRun.state = 'goto_cleaned' AND fakeProcessedImfile.data_state = 'full') +OR + (fakeRun.state = 'goto_scrubbed' AND fakeProcessedImfile.data_state = 'full') +OR + (fakeRun.state = 'goto_purged' AND fakeProcessedImfile.data_state != 'purged')) Index: /branches/eam_branches/20090715/ippTools/share/faketool_pendingcleanuprun.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/faketool_pendingcleanuprun.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/faketool_pendingcleanuprun.sql (revision 25400) @@ -11,3 +11,4 @@ USING (exp_id) WHERE - fakeRun.state = 'goto_cleaned' + (fakeRun.state = 'goto_cleaned' OR fakeRun.state = 'goto_scrubbed' OR fakeRun.state = 'goto_purged') + Index: anches/eam_branches/20090715/ippTools/share/pstamptool_addjob_otherjob.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pstamptool_addjob_otherjob.sql (revision 25399) +++ (revision ) @@ -1,3 +1,0 @@ -INSERT INTO pstampJob - (req_id, rownum, state, jobType, exp_id, outputBase, fault) - VALUES( %lld, '%s', '%s', '%s', %lld, '%s', %s) Index: anches/eam_branches/20090715/ippTools/share/pstamptool_addjob_stampjob.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pstamptool_addjob_stampjob.sql (revision 25399) +++ (revision ) @@ -1,3 +1,0 @@ -INSERT INTO pstampJob - (req_id, rownum, state, jobType, exp_id, outputBase, fault) - VALUES( %lld, '%s', '%s', '%s', %lld, '%s', %s) Index: /branches/eam_branches/20090715/ippTools/share/pstamptool_completedreq.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pstamptool_completedreq.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/pstamptool_completedreq.sql (revision 25400) @@ -0,0 +1,8 @@ +SELECT * +FROM pstampRequest +WHERE state = 'run' + AND ( + SELECT count(*) FROM pstampJob + WHERE pstampJob.req_id = pstampRequest.req_id + AND pstampJob.state != 'stop' + ) = 0 Index: /branches/eam_branches/20090715/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pstamptool_pendingjob.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/pstamptool_pendingjob.sql (revision 25400) @@ -1,4 +1,7 @@ -SELECT - * - FROM pstampJob - WHERE state = 'run' +SELECT pstampJob.* +FROM pstampJob + JOIN pstampRequest using(req_id) +WHERE pstampRequest.state = 'run' + AND pstampRequest.fault = 0 + AND pstampJob.state = 'run' + AND pstampJob.fault = 0 Index: /branches/eam_branches/20090715/ippTools/share/pstamptool_pendingreq.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pstamptool_pendingreq.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/pstamptool_pendingreq.sql (revision 25400) @@ -7,2 +7,3 @@ USING(ds_id) WHERE pstampRequest.state = 'new' + AND pstampRequest.fault = 0 Index: /branches/eam_branches/20090715/ippTools/share/pstamptool_revertjob.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pstamptool_revertjob.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/pstamptool_revertjob.sql (revision 25400) @@ -0,0 +1,5 @@ +UPDATE pstampJob + JOIN pstampRequest USING(req_id) +SET pstampJob.fault = 0 +WHERE pstampRequest.state = 'run' + AND pstampJob.state = 'run' Index: /branches/eam_branches/20090715/ippTools/share/pstamptool_revertreq.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pstamptool_revertreq.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/pstamptool_revertreq.sql (revision 25400) @@ -0,0 +1,4 @@ +UPDATE pstampRequest + SET fault = 0 +WHERE pstampRequest.state != 'stop' + Index: /branches/eam_branches/20090715/ippTools/share/pstamptool_revertreq_deletejobs.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pstamptool_revertreq_deletejobs.sql (revision 25400) +++ /branches/eam_branches/20090715/ippTools/share/pstamptool_revertreq_deletejobs.sql (revision 25400) @@ -0,0 +1,5 @@ +-- delete jobs that got added by an incomplete pstamp parser run +DELETE pstampJob +FROM pstampJob + JOIN pstampRequest USING(req_id) +WHERE pstampRequest.state = 'new' Index: /branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql (revision 25400) @@ -451,4 +451,38 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE addRun ( + add_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + magicked BIGINT, + PRIMARY KEY(add_id), + KEY(add_id), + KEY(cam_id), + KEY(state), + KEY(workdir_state), + KEY(label), + INDEX(add_id, cam_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addProcessedExp ( + add_id BIGINT AUTO_INCREMENT, + dtime_addstar FLOAT, + n_stars INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(add_id), + FOREIGN KEY(add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE fakeRun ( fake_id BIGINT AUTO_INCREMENT, @@ -1086,5 +1120,5 @@ KEY(magic_id), KEY(label), - FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id) + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1407,4 +1441,6 @@ pub_id BIGINT AUTO_INCREMENT, -- link to publishRun path_base VARCHAR(255), -- base path of output + hostname VARCHAR(64), -- name of host + dtime_script FLOAT, -- run time for script fault SMALLINT NOT NULL DEFAULT 0, -- Fault code PRIMARY KEY(pub_id), Index: /branches/eam_branches/20090715/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/eam_branches/20090715/ippTools/share/pxadmin_drop_tables.sql (revision 25399) +++ /branches/eam_branches/20090715/ippTools/share/pxadmin_drop_tables.sql (revision 25400) @@ -19,4 +19,7 @@ DROP TABLE IF EXISTS detRun; DROP TABLE IF EXISTS detInputExp; +DROP TABLE IF EXISTS addRun; +DROP TABLE IF EXISTS addProcessedExp; +DROP TABLE IF EXISTS addMask; DROP TABLE IF EXISTS fakeRun; DROP TABLE IF EXISTS fakeProcessedImfile; Index: /branches/eam_branches/20090715/ippTools/src/Makefile.am =================================================================== --- /branches/eam_branches/20090715/ippTools/src/Makefile.am (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/Makefile.am (revision 25400) @@ -1,3 +1,4 @@ bin_PROGRAMS = \ + addtool \ caltool \ camtool \ @@ -28,4 +29,5 @@ pkginclude_HEADERS = \ + pxadd.h \ pxadmin.h \ pxcam.h \ @@ -42,4 +44,5 @@ noinst_HEADERS = \ + addtool.h \ caltool.h \ camtool.h \ @@ -68,4 +71,5 @@ libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) libpxtools_la_SOURCES = \ + pxadd.c \ pxcam.c \ pxchip.c \ @@ -96,4 +100,10 @@ pstamptool.c \ pstamptoolConfig.c + +addtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +addtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +addtool_SOURCES = \ + addtool.c \ + addtoolConfig.c caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: /branches/eam_branches/20090715/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/addtool.c (revision 25400) +++ /branches/eam_branches/20090715/ippTools/src/addtool.c (revision 25400) @@ -0,0 +1,1116 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupexpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = addtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(ADDTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(ADDTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(ADDTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(ADDTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(ADDTOOL_MODE_BLOCK, blockMode); + MODECASE(ADDTOOL_MODE_MASKED, maskedMode); + MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); + MODECASE(ADDTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(ADDTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(ADDTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + + // find the cam_id of all the exposures that we want to queue up. + psString query = pxDataGet("addtool_find_cam_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + + // loop over our list of addRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + addRunRow *row = addRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into addRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + row->cam_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + "RECIPE", + dvodb ? dvodb : row->dvodb + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set addRun.state to state + if (!pxaddRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set addRun.label to label + if (!pxaddRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +/* static bool pendingimfileMode(pxConfig *config) */ +/* { */ +/* PS_ASSERT_PTR_NON_NULL(config, false); */ + +/* psMetadata *where = psMetadataAlloc(); */ +/* pxaddGetSearchArgs (config, where); */ +/* PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); */ +/* pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "addRun.chip_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "addProcessedExp.class_id", "=="); */ + +/* PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); */ + +/* psString query = pxDataGet("addtool_find_pendingimfile.sql"); */ +/* if (!query) { */ +/* psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); */ +/* return false; */ +/* } */ + +/* // use psDBGenerateWhereSQL because the SQL yields an intermediate table */ +/* if (psListLength(where->list)) { */ +/* psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); */ +/* psStringAppend(&query, " AND %s", whereClause); */ +/* psFree(whereClause); */ +/* } */ +/* psFree(where); */ + +/* if (!p_psDBRunQuery(config->dbh, query)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* psFree(query); */ +/* return false; */ +/* } */ +/* psFree(query); */ + +/* psArray *output = p_psDBFetchResult(config->dbh); */ +/* if (!output) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* if (!psArrayLength(output)) { */ +/* psTrace("addtool", PS_LOG_INFO, "no rows found"); */ +/* psFree(output); */ +/* return true; */ +/* } */ + +/* // negate simple so the default is true */ +/* if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { */ +/* psError(PS_ERR_UNKNOWN, false, "failed to print array"); */ +/* psFree(output); */ +/* return false; */ +/* } */ + +/* psFree(output); */ + +/* return true; */ +/* } */ + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", true, false); + + // optional + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + + PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + addRunRow *pendingRow = addRunObjectFromMetadata(output->data[0]); + psFree(output); + addProcessedExpRow *row = addProcessedExpRowAlloc( + pendingRow->add_id, + dtime_addstar, + n_stars, + path_base, + 0 + ); + + if (!addProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'new' set addRun.state = 'full' + if (!pxaddRunSetState(config, row->add_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); + psFree(row); + psFree(pendingRow); + return false; + } + + // NULL for end_stage means go as far as possible + // EAM : skip here if fault != 0 + // Also, we can run fake even if tess_id is not defined +/* if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "add"))) { */ +/* psFree(row); */ +/* psFree(pendingRow); */ +/* if (!psDBCommit(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* return true; */ +/* } */ + psFree(row); + // else continue on... + +/* if (!pxfakeQueueByAddID(config, */ +/* pendingRow->add_id, */ +/* pendingRow->workdir, */ +/* pendingRow->label, */ +/* pendingRow->reduction, */ +/* pendingRow->expgroup, */ +/* pendingRow->dvodb, */ +/* pendingRow->tess_id, */ +/* pendingRow->end_stage */ +/* )) { */ +/* // rollback */ +/* if (!psDBRollback(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* } */ +/* psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); */ +/* psFree(pendingRow); */ +/* return false; */ +/* } */ + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + return false; + } + + psString query = pxDataGet("addtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + // we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault = 0"); + } + psFree(where); + + // order by add_id so that the postage stamp parser can easliy find the 'latest' astrometry + psStringAppend(&query, " ORDER BY add_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); +/* PXOPT_COPY_S16(config->args, where, "-fault", "addProcessedExp.fault", "=="); */ + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("addtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + + { + psString query = pxDataGet("addtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); +/* PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); */ + + if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!addMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM addMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM addMask WHERE label = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + + psString query = pxDataGet("addtool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (add_id) { + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + } + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_pendingcleanupexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 2; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-add_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + + ExportTable tables [] = { + {"addRun", "addtool_export_run.sql"}, + {"addProcessedExp", "addtool_export_processed_exp.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "addRun")) { + if (!pxSetStateCleaned("addRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + + fprintf (stdout, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); + + psMetadataItem *item = psMetadataLookup (input, "addRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addRunRow *addRun = addRunObjectFromMetadata (entry->data.md); + addRunInsertObject (config->dbh, addRun); + + // fprintf (stdout, "---- add run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + item = psMetadataLookup (input, "addProcessedExp"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + for (int i = 0; i < item->data.list->n; i++) { + psMetadataItem *entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addProcessedExpRow *addProcessedExp = addProcessedExpObjectFromMetadata (entry->data.md); + addProcessedExpInsertObject (config->dbh, addProcessedExp); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + + return true; +} Index: /branches/eam_branches/20090715/ippTools/src/addtool.h =================================================================== --- /branches/eam_branches/20090715/ippTools/src/addtool.h (revision 25400) +++ /branches/eam_branches/20090715/ippTools/src/addtool.h (revision 25400) @@ -0,0 +1,46 @@ +/* + * addtool.h + * + * Copyright (C) 2006 Joshua Hoblitt, Christopher Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef ADDTOOL_H +#define ADDTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + ADDTOOL_MODE_NONE = 0x0, + ADDTOOL_MODE_DEFINEBYQUERY, + ADDTOOL_MODE_UPDATERUN, + ADDTOOL_MODE_PENDINGEXP, + ADDTOOL_MODE_ADDPROCESSEDEXP, + ADDTOOL_MODE_PROCESSEDEXP, + ADDTOOL_MODE_REVERTPROCESSEDEXP, + ADDTOOL_MODE_UPDATEPROCESSEDEXP, + ADDTOOL_MODE_BLOCK, + ADDTOOL_MODE_MASKED, + ADDTOOL_MODE_UNBLOCK, + ADDTOOL_MODE_PENDINGCLEANUPRUN, + ADDTOOL_MODE_PENDINGCLEANUPEXP, + ADDTOOL_MODE_DONECLEANUP, + ADDTOOL_MODE_EXPORTRUN, + ADDTOOL_MODE_IMPORTRUN +} addtoolMode; + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // ADDTOOL_H Index: /branches/eam_branches/20090715/ippTools/src/addtoolConfig.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/addtoolConfig.c (revision 25400) +++ /branches/eam_branches/20090715/ippTools/src/addtoolConfig.c (revision 25400) @@ -0,0 +1,212 @@ +/* + * addtoolConfig.c + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + pxcamSetSearchArgs(updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // XXX is this used? psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(processedexpArgs); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +/* psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); */ + + // -revertprocessedexp + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(revertprocessedexpArgs); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updateprocessedexp + // XXX allow full search options? + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + // XXX allow full search options? + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupexp + // XXX allow full search options? + psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addstar ID", 0); + psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-add_id", 0, "export this addstar ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export tables as cleaned", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change add run properties", ADDTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exps", ADDTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "add a processed exp", ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "show processed exps", ADDTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exp", ADDTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "set a label block", ADDTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupexp", "show exps for cleanup runs", ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", ADDTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", ADDTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "import run from metadata file", ADDTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/20090715/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/camtool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/camtool.c (revision 25400) @@ -656,4 +656,21 @@ return false; } + +/* if (!pxaddQueueByCamID(config, */ +/* pendingRow->cam_id, */ +/* pendingRow->workdir, */ +/* pendingRow->label, */ +/* pendingRow->reduction, */ +/* pendingRow->dvodb */ +/* )) { */ +/* // rollback */ +/* if (!psDBRollback(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* } */ +/* psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun"); */ +/* psFree(pendingRow); */ +/* return false; */ +/* } */ + psFree(pendingRow); Index: /branches/eam_branches/20090715/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/chiptool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/chiptool.c (revision 25400) @@ -54,4 +54,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -93,4 +94,5 @@ MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); @@ -1333,5 +1335,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } - +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) { Index: /branches/eam_branches/20090715/ippTools/src/chiptool.h =================================================================== --- /branches/eam_branches/20090715/ippTools/src/chiptool.h (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/chiptool.h (revision 25400) @@ -45,4 +45,5 @@ CHIPTOOL_MODE_TOFULLIMFILE, CHIPTOOL_MODE_TOPURGEDIMFILE, + CHIPTOOL_MODE_TOSCRUBBEDIMFILE, CHIPTOOL_MODE_EXPORTRUN, CHIPTOOL_MODE_IMPORTRUN Index: /branches/eam_branches/20090715/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/chiptoolConfig.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/chiptoolConfig.c (revision 25400) @@ -248,4 +248,9 @@ psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -282,4 +287,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", CHIPTOOL_MODE_EXPORTRUN, exportrunArgs); Index: /branches/eam_branches/20090715/ippTools/src/dettool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/dettool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/dettool.c (revision 25400) @@ -150,4 +150,5 @@ MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); MODECASE(DETTOOL_MODE_RERUN, rerunMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryMode); // register MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); @@ -1827,4 +1828,15 @@ if (!strcmp(data_state, "drop")) return true; if (!strcmp(data_state, "register")) return true; + // These are valid data states, and are necessary for the cleanup to work correctly. + if (!strcmp(data_state, "full")) return true; + if (!strcmp(data_state, "goto_cleaned")) return true; + if (!strcmp(data_state, "goto_scrubbed")) return true; + if (!strcmp(data_state, "goto_purged")) return true; + if (!strcmp(data_state, "cleaned")) return true; + if (!strcmp(data_state, "scrubbed")) return true; + if (!strcmp(data_state, "purged")) return true; + if (!strcmp(data_state, "error_cleaned")) return true; + if (!strcmp(data_state, "error_scrubbed")) return true; + if (!strcmp(data_state, "error_purged")) return true; psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state); @@ -1853,5 +1865,5 @@ if (!isValidDataState (data_state)) return false; - + char *query = "UPDATE detProcessedImfile SET data_state = '%s'" " WHERE det_id = %" PRId64 @@ -1900,5 +1912,5 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, @@ -1922,6 +1934,7 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration)) { + " AND class_id = '%s'"; +/* fprintf(stderr,"DETTOOL SAYS: %s\n",query); */ + if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration,class_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for det_id %" PRId64 ", iteration %" PRId32, @@ -1944,5 +1957,5 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, Index: /branches/eam_branches/20090715/ippTools/src/dettool.h =================================================================== --- /branches/eam_branches/20090715/ippTools/src/dettool.h (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/dettool.h (revision 25400) @@ -111,4 +111,5 @@ DETTOOL_MODE_REVERTDETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUNSUMMARY, + DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUN, DETTOOL_MODE_RERUN, @@ -209,4 +210,5 @@ bool revertdetrunsummaryMode(pxConfig *config); bool updatedetrunsummaryMode(pxConfig *config); +bool pendingcleanup_detrunsummaryMode(pxConfig *config); // other utilities Index: /branches/eam_branches/20090715/ippTools/src/dettoolConfig.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/dettoolConfig.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/dettoolConfig.c (revision 25400) @@ -41,11 +41,11 @@ // XXX EAM : is this used? does it make sense? psMetadata *pendingArgs = psMetadataAlloc(); - psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, "search by URL", NULL); - psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definebytag @@ -125,4 +125,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); @@ -137,4 +138,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max", 0, "define max moon phase", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); @@ -207,7 +209,7 @@ // -addcorrectimfile psMetadata *addcorrectimfileArgs = psMetadataAlloc(); - psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); @@ -306,5 +308,5 @@ psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedimfile @@ -366,5 +368,5 @@ psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); - psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedexp @@ -426,5 +428,5 @@ psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); - psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_stacked @@ -478,5 +480,5 @@ psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedstat @@ -540,5 +542,5 @@ psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedimfile @@ -598,5 +600,5 @@ psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); - psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedexp @@ -678,5 +680,5 @@ psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residimfile @@ -759,5 +761,5 @@ psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); - psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residexp @@ -813,4 +815,13 @@ psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "reject this detrun iteration as a master", false); + psMetadataAddStr(updatedetrunsummaryArgs, PS_LIST_TAIL, "-data_state", 0, "set the data state", NULL); + psMetadataAddS32(updatedetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -pendingcleanup_detrunsummary + psMetadata *pendingcleanup_detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddU64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updatedetrun @@ -968,4 +979,6 @@ PXOPT_ADD_MODE("-updatedetrunsummary", "", DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryArgs); + PXOPT_ADD_MODE("-pendingcleanup_detrunsummary", "", DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryArgs); + PXOPT_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); PXOPT_ADD_MODE("-rerun", "", DETTOOL_MODE_RERUN, rerunArgs); Index: /branches/eam_branches/20090715/ippTools/src/dettool_detrunsummary.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/dettool_detrunsummary.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/dettool_detrunsummary.c (revision 25400) @@ -336,5 +336,65 @@ } -// XXX need to add -data_state here +bool pendingcleanup_detrunsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_detrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + // fprintf(stderr,">>>%s<<<\n",query); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingCleanupDetRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return(false); + } + psFree(output); + + return(true); +} + + +// preliminary code now. bool updatedetrunsummaryMode(pxConfig *config) { @@ -344,4 +404,7 @@ PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", ((accept == 0)&&(reject == 0)), false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", ((accept == 0)&&(reject == 0)), false); + if (accept && reject) { @@ -350,13 +413,109 @@ } - if (!(accept || reject)) { - psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required"); - return false; - } - - char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; - if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + if (!(accept || reject || (data_state != NULL))) { + psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required if -data_state is not supplied"); + return false; + } + + if (accept || reject) { + char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + else { + PS_ASSERT_PTR_NON_NULL(data_state,false); + +/* if (!isValidDataState(data_state)) return false; */ + + + + char *query_detRunSummary = "UPDATE detRunSummary SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detRunSummary, data_state, det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return false; + } + + /* This check allows the one update to flag everything for cleanup. The check for full is only temporary while I test for bugs. */ + if ((!strncmp(data_state,"goto_",5) + // || (!strcmp(data_state,"full")) + // || (!strcmp(data_state,"cleaned")) + )) { + char *query_detProcessedImfile = "UPDATE detProcessedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedImfile,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detProcessedExp = "UPDATE detProcessedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedExp,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedImfile = "UPDATE detNormalizedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedStatImfile = "UPDATE detNormalizedStatImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedStatImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedStatImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedExp = "UPDATE detNormalizedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidImfile = "UPDATE detResidImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidExp = "UPDATE detResidExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detStackedImfile = "UPDATE detStackedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detStackedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detStackedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + } + /* End if */ + } Index: /branches/eam_branches/20090715/ippTools/src/dettool_processedimfile.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/dettool_processedimfile.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/dettool_processedimfile.c (revision 25400) @@ -314,5 +314,5 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - +/* int i; */ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); @@ -345,4 +345,5 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",query); */ psFree(query); @@ -350,8 +351,10 @@ if (!output) { psError(PS_ERR_UNKNOWN, false, "database error"); +/* fprintf(stderr,"WTF !output?\n"); */ return false; } if (!psArrayLength(output)) { psTrace("dettool", PS_LOG_INFO, "no rows found"); +/* fprintf(stderr,"WTF no rows??\n"); */ psFree(output); return true; @@ -359,4 +362,6 @@ // negative simple so the default is true +/* i = (int) ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple); */ +/* fprintf(stderr,">>%d<<\n",i); */ if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -364,4 +369,6 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",output); */ +/* psFree(output); */ psFree(output); Index: /branches/eam_branches/20090715/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/difftool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/difftool.c (revision 25400) @@ -584,7 +584,7 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -1179,6 +1179,6 @@ if (!available && (num != skycell_count)) { - psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" - PRId64 " need %" PRId64, num, skycell_count, warp_id); + psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64 + " need %" PRId64, num, warp_id, skycell_count); continue; } Index: /branches/eam_branches/20090715/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/disttool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/disttool.c (revision 25400) @@ -384,6 +384,9 @@ psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage); - if (set_label == NULL) { - set_label = target_label; + psString new_label; + if (set_label != NULL) { + new_label = set_label; + } else { + new_label = target_label; } distRunRow *row = distRunRowAlloc( @@ -392,5 +395,5 @@ stage, stage_id, - set_label, + new_label, outroot, clean, Index: /branches/eam_branches/20090715/ippTools/src/faketool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/faketool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/faketool.c (revision 25400) @@ -52,4 +52,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -91,4 +92,5 @@ MODECASE(FAKETOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(FAKETOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(FAKETOOL_MODE_IMPORTRUN, importrunMode); @@ -1251,4 +1253,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) Index: /branches/eam_branches/20090715/ippTools/src/faketool.h =================================================================== --- /branches/eam_branches/20090715/ippTools/src/faketool.h (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/faketool.h (revision 25400) @@ -45,4 +45,5 @@ FAKETOOL_MODE_TOFULLIMFILE, FAKETOOL_MODE_TOPURGEDIMFILE, + FAKETOOL_MODE_TOSCRUBBEDIMFILE, FAKETOOL_MODE_EXPORTRUN, FAKETOOL_MODE_IMPORTRUN Index: /branches/eam_branches/20090715/ippTools/src/faketoolConfig.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/faketoolConfig.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/faketoolConfig.c (revision 25400) @@ -301,4 +301,9 @@ psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "fake ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -337,4 +342,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", FAKETOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", FAKETOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", FAKETOOL_MODE_IMPORTRUN, importrunArgs); Index: /branches/eam_branches/20090715/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pstamptool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/pstamptool.c (revision 25400) @@ -44,4 +44,5 @@ static bool pendingjobMode(pxConfig *config); static bool updatejobMode(pxConfig *config); +static bool revertjobMode(pxConfig *config); static bool addprojectMode(pxConfig *config); static bool projectMode(pxConfig *config); @@ -80,4 +81,5 @@ MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode); MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode); + MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode); MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode); MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); @@ -224,21 +226,19 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - // PXOPT_LOOKUP_STR(outFileset, config->args, "-out_fileset", true, false); - PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); - - char *query ="INSERT INTO pstampRequest" - " (state, uri, ds_id, fault)" - " VALUES( 'new', '%s', %" PRId64 ", 0 )"; - if (!p_psDBRunQueryF(config->dbh, query, uri, ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psU64 affected = psDBAffectedRows(config->dbh); - if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, - "should have affected one row but %" PRIu64 " rows were modified", - affected); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); + + if (!pstampRequestInsert(config->dbh, + 0, // req_id + ds_id, + "new", //state + name, + NULL, // reqType + NULL, // outProduct + uri, + 0 // fault + )) { + psError(PS_ERR_UNKNOWN, false, "failed to insert request"); return false; } @@ -315,10 +315,23 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = NULL; - psStringAppend(&query, "SELECT * from pstampRequest WHERE req_id = %" PRId64, req_id); + if (!psListLength(where->list)) { + psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); + return false; + } + + psString query = psStringCopy("SELECT * from pstampRequest"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); // treat limit == 0 as "no limit" @@ -342,7 +355,10 @@ } if (!psArrayLength(output)) { - psTrace("pstamptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; + psTrace("pstamptool", PS_LOG_INFO, "request not found"); + // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for + psError(PXTOOLS_ERR_DATA, true, "request not found"); + psFree(output); + // we return false so that the caller can determine that a request does not exist + return false; } @@ -366,8 +382,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = - "SELECT * FROM pstampRequest WHERE state = 'run' AND " - "(SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id " - " AND pstampJob.state != 'stop') = 0" ; + psString query = pxDataGet("pstamptool_completedreq.sql"); // treat limit == 0 as "no limit" @@ -476,17 +489,40 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); - - // printf("Revert request %" PRId64 "\n", req_id); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + + // delete any jobs that were queued by requests that didn't complete parsing (pstampRequest.state = 'new' + // If state = 'run' was supplied this will be a no-op + psString query = pxDataGet("pstamptool_revertreq_deletejobs.sql"); + psStringAppend(&query, " AND %s", whereClause); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + // clear fault for requests + query = pxDataGet("pstamptool_revertreq.sql"); + psStringAppend(&query, " AND %s", whereClause); - if (!p_psDBRunQueryF(config->dbh, "DELETE FROM pstampJob where req_id = %" PRId64, req_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!p_psDBRunQueryF(config->dbh, - "UPDATE pstampRequest set state ='new', name=NULL, reqType=NULL, fault=0 where req_id = %" PRId64, req_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); return true; @@ -496,5 +532,4 @@ { bool stampJob = false; - char *query = NULL; PS_ASSERT_PTR_NON_NULL(config, false); @@ -503,11 +538,17 @@ PXOPT_LOOKUP_STR(rownum, config->args, "-rownum", true, false); PXOPT_LOOKUP_STR(job_type, config->args, "-job_type", false, false); - PXOPT_LOOKUP_STR(outputBase, config->args, "-outputBase", true, false); + PXOPT_LOOKUP_STR(outputBase, config->args, "-outputBase", false, false); PXOPT_LOOKUP_STR(stateString, config->args, "-state", false, false); - PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + // unless the job is being inserted with stop state require outputBase + if (strcmp(stateString, "stop") && !outputBase) { + psError(PS_ERR_UNKNOWN, true, "-outputBase is required"); + return false; + } + // default value for job_type is defined in pstamptoolConfig.c - if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query")) { + if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query") || !strcmp(job_type, "none")) { stampJob = false; } else if (!strcmp(job_type, "stamp")) { @@ -517,15 +558,14 @@ return false; } - if (stampJob) { - query = pxDataGet("pstamptool_addjob_stampjob.sql"); - } else { - query = pxDataGet("pstamptool_addjob_otherjob.sql"); - } - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, req_id, rownum, stateString, job_type, exp_id, outputBase, fault)) { + if (!pstampJobInsert(config->dbh, + 0, // job_id + req_id, + rownum, + stateString, + job_type, + fault, + exp_id, + outputBase + )) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -715,4 +755,34 @@ return true; } + +static bool revertjobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("pstamptool_revertjob.sql"); + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} static bool addprojectMode(pxConfig *config) { Index: /branches/eam_branches/20090715/ippTools/src/pstamptool.h =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pstamptool.h (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/pstamptool.h (revision 25400) @@ -39,4 +39,5 @@ PSTAMPTOOL_MODE_JOBRESULT, PSTAMPTOOL_MODE_UPDATEJOB, + PSTAMPTOOL_MODE_REVERTJOB, PSTAMPTOOL_MODE_ADDPROJECT, PSTAMPTOOL_MODE_MODPROJECT, Index: /branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c (revision 25400) @@ -63,7 +63,7 @@ // -addreq psMetadata *addreqArgs = psMetadataAlloc(); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); - psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); - // psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-out_fileset", 0, "define request output_fileset", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0, "define request name", NULL); // -pendingreq @@ -75,7 +75,9 @@ // -listreq psMetadata *listreqArgs = psMetadataAlloc(); - psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id (required)", 0); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "list by req_id", 0); + psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0, "list by name", NULL); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0, "req_id to not list", 0); psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -completedreq @@ -96,5 +98,7 @@ // -revertreq psMetadata *revertreqArgs = psMetadataAlloc(); - psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to revert", 0); + psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS16(revertreqArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); // -addjob @@ -106,5 +110,5 @@ psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "new state", "run"); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "exposure id", 0); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "new result", 0); // -listjob @@ -127,4 +131,10 @@ psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + + // -revertjob + psMetadata *revertjobArgs = psMetadataAlloc(); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id to revert", 0); + psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); // -addproject @@ -158,5 +168,5 @@ PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); - PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); + PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); PXOPT_ADD_MODE("-listreq", "", PSTAMPTOOL_MODE_LISTREQ, listreqArgs); PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); @@ -166,5 +176,6 @@ PXOPT_ADD_MODE("-listjob", "", PSTAMPTOOL_MODE_LISTJOB, listjobArgs); PXOPT_ADD_MODE("-pendingjob", "", PSTAMPTOOL_MODE_PENDINGJOB, pendingjobArgs); - PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + PXOPT_ADD_MODE("-revertjob", "", PSTAMPTOOL_MODE_REVERTJOB, revertjobArgs); PXOPT_ADD_MODE("-adddatastore", "", PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreArgs); Index: /branches/eam_branches/20090715/ippTools/src/pubtool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pubtool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/pubtool.c (revision 25400) @@ -259,7 +259,9 @@ // required PXOPT_LOOKUP_S64(pub_id, config->args, "-pub_id", true, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); // optional + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); @@ -269,5 +271,5 @@ } - if (!publishDoneInsert(config->dbh, pub_id, path_base, fault)) { + if (!publishDoneInsert(config->dbh, pub_id, path_base, hostname, dtime_script, fault)) { psError(PS_ERR_UNKNOWN, false, "Unable to add file"); if (!psDBRollback(config->dbh)) { @@ -332,5 +334,7 @@ psFree(query); - return true; -} - + psLogMsg("pubtool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + + return true; +} + Index: /branches/eam_branches/20090715/ippTools/src/pubtoolConfig.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pubtoolConfig.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/pubtoolConfig.c (revision 25400) @@ -68,4 +68,6 @@ psMetadataAddS64(addArgs, PS_LIST_TAIL, "-pub_id", 0, "define pub_id (required)", 0); psMetadataAddStr(addArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL); + psMetadataAddStr(addArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddF32(addArgs, PS_LIST_TAIL, "-dtime_script", 0, "define time for script", NAN); psMetadataAddS32(addArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); Index: /branches/eam_branches/20090715/ippTools/src/pxadd.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pxadd.c (revision 25400) +++ /branches/eam_branches/20090715/ippTools/src/pxadd.c (revision 25400) @@ -0,0 +1,262 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" + +bool pxaddSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); */ + + return true; +} + +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); +/* PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); */ +/* PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); */ +/* PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); */ + + return true; +} + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE addRun SET state = '%s', magicked = %" PRId64 " WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + + +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE addRun SET addRun.label = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, label, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + +// Need to think more about this to see what we want it to do. BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("addtool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + // fprintf(stderr,"%s",query); + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQueryF(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + label ? label : "NULL", + dvodb ? dvodb : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: /branches/eam_branches/20090715/ippTools/src/pxadd.h =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pxadd.h (revision 25400) +++ /branches/eam_branches/20090715/ippTools/src/pxadd.h (revision 25400) @@ -0,0 +1,44 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADD_H +#define PXADD_H 1 + +#include + +#include "pxtools.h" + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked); +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxaddSetSearchArgs (psMetadata *md); +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where); + +// Likely BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb); + + +#endif // PXADD_H Index: /branches/eam_branches/20090715/ippTools/src/pxtools.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pxtools.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/pxtools.c (revision 25400) @@ -37,4 +37,6 @@ if (!strcmp(state, "goto_cleaned")) return true; if (!strcmp(state, "error_cleaned")) return true; + if (!strcmp(state, "goto_purged")) return true; + if (!strcmp(state, "error_purged")) return true; if (!strcmp(state, "goto_scrubbed")) return true; if (!strcmp(state, "error_scrubbed")) return true; @@ -42,12 +44,11 @@ if (!strcmp(state, "update")) return true; if (!strcmp(state, "purged")) return true; - if (!strcmp(state, "goto_purged")) return true; - if (!strcmp(state, "error_purged")) return true; - + if (!strcmp(state, "scrubbed")) return true; return false; } -// 'scrubbed' is a virtual state equivalent to cleaned, but allows files to be removed -// even if the config files is missing +// 'scrubbed' is no longer a virtual state equivalent to cleaned, but allows files to be removed +// even if the config files is missing. This change was prompted as files that are cleaned can +// be regenerated, but that is not certain after being scrubbed. Index: /branches/eam_branches/20090715/ippTools/src/pxtools.h =================================================================== --- /branches/eam_branches/20090715/ippTools/src/pxtools.h (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/pxtools.h (revision 25400) @@ -37,4 +37,5 @@ #include "pxtoolsErrorCodes.h" +#include "pxadd.h" #include "pxcam.h" #include "pxchip.h" Index: /branches/eam_branches/20090715/ippTools/src/regtool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/regtool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/regtool.c (revision 25400) @@ -331,4 +331,5 @@ PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false); psString query = pxDataGet("regtool_processedimfile.sql"); @@ -352,4 +353,9 @@ // don't list faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); + } + + // add the ORDER BY statement if desired + if (ordered_by_date) { + psStringAppend(&query, " ORDER BY dateobs"); } Index: /branches/eam_branches/20090715/ippTools/src/warptool.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/warptool.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/warptool.c (revision 25400) @@ -53,4 +53,5 @@ static bool tocleanedskyfileMode(pxConfig *config); static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); static bool tofullskyfileMode(pxConfig *config); static bool updateskyfileMode(pxConfig *config); @@ -102,4 +103,5 @@ MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); @@ -1668,4 +1670,8 @@ return change_skyfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); +} static bool updateskyfileMode(pxConfig *config) Index: /branches/eam_branches/20090715/ippTools/src/warptool.h =================================================================== --- /branches/eam_branches/20090715/ippTools/src/warptool.h (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/warptool.h (revision 25400) @@ -49,4 +49,5 @@ WARPTOOL_MODE_TOCLEANEDSKYFILE, WARPTOOL_MODE_TOPURGEDSKYFILE, + WARPTOOL_MODE_TOSCRUBBEDSKYFILE, WARPTOOL_MODE_TOFULLSKYFILE, WARPTOOL_MODE_UPDATESKYFILE, Index: /branches/eam_branches/20090715/ippTools/src/warptoolConfig.c =================================================================== --- /branches/eam_branches/20090715/ippTools/src/warptoolConfig.c (revision 25399) +++ /branches/eam_branches/20090715/ippTools/src/warptoolConfig.c (revision 25400) @@ -279,4 +279,9 @@ psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + // -toscrubbedskyfile + psMetadata *toscrubbedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + // -tofullskyfile psMetadata *tofullskyfileArgs = psMetadataAlloc(); @@ -328,4 +333,5 @@ PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs);