Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/Makefile.am (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/Makefile.am (revision 37068) @@ -105,4 +105,5 @@ detselect_search.sql \ detselect_select.sql \ + detselect_show.sql \ dettool_addprocessedexp.sql \ dettool_childlessrun.sql \ @@ -189,4 +190,5 @@ disttool_definebyquery_diff.sql \ disttool_definebyquery_fake.sql \ + disttool_definebyquery_ff.sql \ disttool_definebyquery_raw.sql \ disttool_definebyquery_raw_no_magic.sql \ @@ -206,4 +208,5 @@ disttool_pending_diff.sql \ disttool_pending_fake.sql \ + disttool_pending_ff.sql \ disttool_pending_raw.sql \ disttool_pending_sky.sql \ @@ -400,4 +403,5 @@ stacktool_tosummary.sql \ stacktool_addsummary.sql \ + stacktool_revertsummary.sql \ stacktool_summary.sql \ staticskytool_definebyquery_select.sql \ @@ -487,4 +491,21 @@ releasetool_definerelgroup_select_exp_lap.sql \ releasetool_stacksummary.sql \ - releasetool_pendingrelgroup.sql - + releasetool_pendingrelgroup.sql \ + fftool_definebyquery.sql \ + fftool_definebyquery_select_warps.sql \ + fftool_result.sql \ + fftool_revert.sql \ + fftool_revertsummary.sql \ + fftool_summary.sql \ + fftool_todo.sql \ + fftool_toadvance.sql \ + fftool_export_run.sql \ + fftool_export_input.sql \ + fftool_export_result.sql \ + fftool_export_summary.sql \ + remotetool_definebyquery_chip.sql \ + remotetool_definebyquery_cam.sql \ + remotetool_definebyquery_warp.sql \ + remotetool_definebyquery_stack.sql \ + remotetool_dropcomponent.sql \ + remotetool_updatepoll.sql Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/camtool_find_processedexp.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/camtool_find_processedexp.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/camtool_find_processedexp.sql (revision 37068) @@ -1,4 +1,5 @@ SELECT camProcessedExp.*, + HEX(IFNULL(camProcessedExp.astrom_chips, 0)) AS good_astrom_chips, camRun.workdir, camRun.label, Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/detselect_show.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/detselect_show.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/detselect_show.sql (revision 37068) @@ -0,0 +1,35 @@ +-- this query needs to use the same fields in both of the tables in +-- the union statement, but we need to report the +-- detRunSummary.iteration in the first case, and this is missing in +-- the second case. + +SELECT DISTINCT + det_id, + good_iteration as iteration, + filelevel, + det_type, + mode, + registered, + filter, + time_begin, + time_end +FROM + (SELECT DISTINCT + detRun.*, + detRunSummary.iteration as good_iteration + FROM detRun + JOIN detRunSummary + USING(det_id) + WHERE + detRun.mode = 'master' + AND detRunSummary.accept = 1 + UNION + SELECT DISTINCT + detRun.*, + detRun.iteration as good_iteration + FROM detRun + WHERE + (detRun.mode = 'register' OR detRun.mode = 'correction') + ) as Foo +WHERE + (state = 'stop') Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_SSdiff.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_SSdiff.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_SSdiff.sql (revision 37068) @@ -22,5 +22,3 @@ AND distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL - AND ((diffRun.state = 'full') OR (distTarget.clean AND diffRun.state = 'cleaned')) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_camera.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_camera.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_camera.sql (revision 37068) @@ -21,4 +21,3 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL -- no existing distRun AND ((camRun.state = 'full') OR (distTarget.clean AND camRun.state = 'cleaned')) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_chip.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_chip.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_chip.sql (revision 37068) @@ -20,4 +20,3 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL -- no existing distRun AND ((chipRun.state = 'full') OR (distTarget.clean AND chipRun.state = 'cleaned')) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_chip_bg.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_chip_bg.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_chip_bg.sql (revision 37068) @@ -23,4 +23,3 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL -- no existing distRun AND ((chipBackgroundRun.state = 'full') OR (distTarget.clean AND chipBackgroundRun.state = 'cleaned')) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_diff.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_diff.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_diff.sql (revision 37068) @@ -25,5 +25,4 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL AND ((diffRun.state = 'full') OR (distTarget.clean AND diffRun.state = 'cleaned')) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_fake.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_fake.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_fake.sql (revision 37068) @@ -20,4 +20,3 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL AND ((fakeRun.state = 'full') OR (distTarget.clean AND fakeRun.state = 'cleaned')) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_ff.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_ff.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_ff.sql (revision 37068) @@ -0,0 +1,26 @@ +SELECT DISTINCT + 'ff' as stage, + fullForceRun.ff_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'ff.$skycell_id.$ff_id' + CONCAT_WS('.', 'ff', stackRun.skycell_id, convert(fullForceRun.ff_id, CHAR)) as run_tag, + fullForceRun.label, + fullForceRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM fullForceRun +JOIN skycalRun USING(skycal_id) +JOIN stackRun using(stack_id) +JOIN distTarget ON distTarget.stage = 'ff' + AND fullForceRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = ff_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND fullForceRun.state = 'full' + AND stackRun.filter = distTarget.filter + -- we shouldn't need to check fault. If faulted it shouldn't be full + -- AND (skycalResult.fault = 0 AND skycalResult.quality = 0) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_raw.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_raw.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_raw.sql (revision 37068) @@ -16,6 +16,5 @@ JOIN rcInterest USING(target_id) LEFT JOIN distRun ON distRun.stage_id = exp_id AND distRun.target_id = distTarget.target_id -WHERE distRun.dist_id IS NULL -- no existing distRun for this exposure - AND distTarget.state = 'enabled' -- target and intrest are enabled +WHERE distTarget.state = 'enabled' -- target and intrest are enabled AND rcInterest.state = 'enabled' AND magicDSRun.state = 'full' -- destreaked files are available Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_sky.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_sky.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_sky.sql (revision 37068) @@ -23,5 +23,4 @@ AND rcInterest.state = 'enabled' AND distTarget.filter = 'multi' - AND distRun.dist_id IS NULL AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned')) -- we shouldn't need to check fault. If faulted it shouldn't be full Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_sky_singlefilter.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_sky_singlefilter.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_sky_singlefilter.sql (revision 37068) @@ -22,5 +22,4 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned')) AND stackRun.filter = distTarget.filter Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_skycal.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_skycal.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_skycal.sql (revision 37068) @@ -1,3 +1,3 @@ -SELECT +SELECT DISTINCT 'skycal' as stage, skycalRun.skycal_id AS stage_id, @@ -21,5 +21,4 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL AND ((skycalRun.state = 'full') OR (distTarget.clean AND skycalRun.state = 'cleaned')) AND stackRun.filter = distTarget.filter Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_stack.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_stack.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_stack.sql (revision 37068) @@ -21,5 +21,4 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL AND ((stackRun.state = 'full') OR (distTarget.clean AND stackRun.state = 'cleaned')) -- we shouldn't need to check fault. If faulted it shouldn't be full Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_warp.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_warp.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_warp.sql (revision 37068) @@ -22,5 +22,4 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL AND ((warpRun.state = 'full') OR (distTarget.clean AND warpRun.state = 'cleaned')) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_warp_bg.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_warp_bg.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_warp_bg.sql (revision 37068) @@ -24,5 +24,4 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' - AND distRun.dist_id IS NULL AND ((warpBackgroundRun.state = 'full') OR (distTarget.clean AND warpBackgroundRun.state = 'cleaned')) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_pending_ff.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_pending_ff.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_pending_ff.sql (revision 37068) @@ -0,0 +1,86 @@ +-- look for fullForceResult components and a summary components +SELECT * FROM +( +SELECT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + CONCAT_WS('.', warp_id, stackRun.skycell_id) AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + fullForceResult.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + fullForceRun.state, + 'full' AS data_state, + fullForceResult.quality, + distRun.no_magic, + warpRun.magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN fullForceRun ON stage_id = ff_id +JOIN skycalRun USING(skycal_id) +JOIN stackRun USING(stack_id) +JOIN fullForceResult USING(ff_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND CONCAT_WS('.', warp_id, stackRun.skycell_id) = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'ff' + AND distComponent.dist_id IS NULL + AND (Label.active OR Label.active IS NULL) +UNION +SELECT DISTINCT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + 'summary' AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + fullForceSummary.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + fullForceRun.state, + 'full' AS data_state, + fullForceSummary.quality, + distRun.no_magic, + 0 AS magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN fullForceRun ON stage_id = ff_id +JOIN fullForceSummary USING(ff_id) +JOIN skycalRun USING(skycal_id) +JOIN stackRun USING(stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun ON camRun.chip_id = chipRun.chip_id +JOIN rawExp using(exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND 'summary' = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'ff' + AND distComponent.dist_id IS NULL + AND (Label.active OR Label.active IS NULL) +) as distRun +-- where arguments are appended here +WHERE 1 Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_toadvance.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_toadvance.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_toadvance.sql (revision 37068) @@ -308,3 +308,33 @@ COUNT(diffSkyfile.skycell_id) = COUNT(distComponent.component) AND SUM(distComponent.fault) = 0 +UNION +-- ff stage +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + distRun.label, + clean + -- ,COUNT(warp_id) + -- ,COUNT(ff_id) + FROM distRun + JOIN fullForceRun ON stage_id = ff_id + JOIN fullForceResult USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND distComponent.component = CONCAT_WS('.', warp_id, stackRun.skycell_id) + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'ff' + GROUP BY + distRun.dist_id, + ff_id + HAVING + -- number of dist components is the number of warps plus 1 (for the summary component) + COUNT(fullForceResult.ff_id) = COUNT(distComponent.component) + AND SUM(distComponent.fault) = 0 ) as Foo Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_definebyquery.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_definebyquery.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_definebyquery.sql (revision 37068) @@ -0,0 +1,14 @@ +SELECT + skycalRun.skycal_id, + skycalResult.path_base, + skycalRun.data_group, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter +FROM skycalRun + JOIN skycalResult USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) + -- join hook %s +WHERE + skycalRun.state = 'full' AND skycalResult.quality = 0 Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_definebyquery_select_warps.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_definebyquery_select_warps.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_definebyquery_select_warps.sql (revision 37068) @@ -0,0 +1,9 @@ +SELECT warp_id +FROM warpRun join warpSkyfile using(warp_id, tess_id) + JOIN fakeRun using(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN skycell ON skycell.tess_id = warpSkyfile.tess_id AND skycell.skycell_id = warpSkyfile.skycell_id +WHERE warpRun.tess_id = '%s' AND warpSkyfile.skycell_id = '%s' AND rawExp.filter = '%s' + AND warpRun.state = 'full' AND warpSkyfile.quality = 0 and warpSkyfile.fault = 0 Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_input.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_input.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_input.sql (revision 37068) @@ -0,0 +1,3 @@ +SELECT + fullForceInput.* +FROM fullForceInput Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_result.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_result.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_result.sql (revision 37068) @@ -0,0 +1,3 @@ +SELECT + fullForceResult.* +FROM fullForceResult Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_run.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_run.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_run.sql (revision 37068) @@ -0,0 +1,3 @@ +SELECT + fullForceRun.* +FROM fullForceRun Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_summary.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_summary.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_export_summary.sql (revision 37068) @@ -0,0 +1,3 @@ +SELECT + fullForceSummary.* +FROM fullForceSummary Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_result.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_result.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_result.sql (revision 37068) @@ -0,0 +1,23 @@ +SELECT + fullForceResult.*, + fullForceRun.state, + fullForceRun.label, + fullForceRun.data_group, + fullForceRun.skycal_id, + fullForceRun.workdir, + fullForceRun.sources_path_base, + CONCAT_WS('.', warp_id, skycell_id) AS dist_component, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceResult USING(ff_id) + JOIN skycalRun using(skycal_id) + JOIN stackRun using(stack_id) + JOIN warpRun using(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun using(fake_id) + JOIN camRun using(cam_id) + JOIN chipRun using(chip_id) + JOIN rawExp using(exp_id) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_revert.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_revert.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_revert.sql (revision 37068) @@ -0,0 +1,7 @@ +DELETE FROM fullForceResult +USING fullForceResult, fullForceRun, skycalRun, stackRun, skycell +WHERE fullForceRun.ff_id = fullForceResult.ff_id + AND fullForceResult.fault != 0 + AND fullForceRun.skycal_id = skycalRun.skycal_id + AND skycalRun.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_revertsummary.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_revertsummary.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_revertsummary.sql (revision 37068) @@ -0,0 +1,7 @@ +DELETE FROM fullForceSummary +USING fullForceSummary, fullForceRun, skycalRun, stackRun, skycell +WHERE fullForceRun.ff_id = fullForceSummary.ff_id + AND fullForceSummary.fault != 0 + AND fullForceRun.skycal_id = skycalRun.skycal_id + AND skycalRun.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_summary.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_summary.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_summary.sql (revision 37068) @@ -0,0 +1,24 @@ +SELECT DISTINCT + fullForceSummary.*, + fullForceRun.state, + fullForceRun.label, + fullForceRun.data_group, + fullForceRun.skycal_id, + fullForceRun.workdir, + fullForceRun.sources_path_base, + 'summary' AS dist_component, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceSummary USING(ff_id) + JOIN fullForceInput USING(ff_id) + JOIN skycalRun using(skycal_id) + JOIN stackRun using(stack_id) + JOIN warpRun using(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun using(fake_id) + JOIN camRun using(cam_id) + JOIN chipRun using(chip_id) + JOIN rawExp using(exp_id) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_summaryinputs.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_summaryinputs.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_summaryinputs.sql (revision 37068) @@ -0,0 +1,4 @@ +SELECT + fullForceResult.* +FROM fullForceRun + JOIN fullForceResult USING(ff_id) Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_toadvance.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_toadvance.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_toadvance.sql (revision 37068) @@ -0,0 +1,22 @@ +SELECT + fullForceRun.*, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceInput USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN warpRun USING(warp_id, tess_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN fullForceResult USING(ff_id, warp_id) + LEFT JOIN fullForceSummary USING(ff_id) +WHERE fullForceRun.state = 'new' AND fullForceSummary.ff_id IS NULL + -- WHERE hook %s + GROUP BY ff_id + HAVING COUNT(fullForceInput.warp_id) = COUNT(fullForceResult.warp_id) + AND SUM(fullForceResult.fault) = 0 Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_todo.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_todo.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/fftool_todo.sql (revision 37068) @@ -0,0 +1,22 @@ +SELECT + fullForceRun.*, + fullForceInput.warp_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera, + warpSkyfile.path_base AS warp_path_base +FROM fullForceRun + JOIN fullForceInput USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN warpRun USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN fullForceResult USING(ff_id, warp_id) +WHERE fullForceRun.state = 'new' + AND fullForceResult.ff_id IS NULL + AND warpSkyfile.data_state = 'full' Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/pxadmin_create_tables.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/pxadmin_create_tables.sql (revision 37068) @@ -523,4 +523,5 @@ quality SMALLINT NOT NULL DEFAULT 0, background_model SMALLINT, + astrom_chips BIGINT, PRIMARY KEY(cam_id), KEY(fault), @@ -2307,4 +2308,84 @@ ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +CREATE TABLE fullForceRun ( + ff_id BIGINT NOT NULL AUTO_INCREMENT, + skycal_id BIGINT, + sources_path_base VARCHAR(255), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + note VARCHAR(255), + reduction VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(ff_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(skycal_id), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceInput ( + ff_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(ff_id, warp_id), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceResult ( + ff_id BIGINT, + warp_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id, warp_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceSummary ( + ff_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE remoteRun ( + remote_id BIGINT NOT NULL AUTO_INCREMENT, + state VARCHAR(64) NOT NULL, + stage VARCHAR(64) NOT NULL, + label VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + job_id BIGINT, + last_poll TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + PRIMARY KEY (remote_id), + KEY (state), + KEY (stage), + KEY (label), + KEY (job_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE remoteComponent ( + remote_id BIGINT, + stage_id BIGINT, + KEY (stage_id), + FOREIGN KEY (remote_id) REFERENCES remoteRun(remote_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; -- These comment lines are here to avoid an empty query error. Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/pxadmin_drop_tables.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/pxadmin_drop_tables.sql (revision 37068) @@ -120,4 +120,9 @@ DROP TABLE IF EXISTS dqstatsContent; DROP TABLE IF EXISTS dqstatsRun; +DROP TABLE IF EXISTS fullForceRun; +DROP TABLE IF EXISTS fullForceInput; +DROP TABLE IF EXISTS fullForceResult; +DROP TABLE IF EXISTS fullForceSummary; + SET FOREIGN_KEY_CHECKS=1 Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_cam.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_cam.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_cam.sql (revision 37068) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT cam_id AS stage_id + FROM camRun +WHERE +camRun.state = 'new' +-- where hook %s +GROUP BY cam_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING (remote_id) +WHERE +remoteRun.stage = 'camera' +-- where hook %s +) AS R +USING (stage_id) +WHERE R.remote_id IS NULL Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_chip.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_chip.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_chip.sql (revision 37068) @@ -0,0 +1,26 @@ +SELECT stage_id FROM + +( SELECT chip_id AS stage_id + FROM chipRun +WHERE +chipRun.state = 'new' +-- where hook %s +GROUP BY chip_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING (remote_id) +WHERE +remoteRun.stage = 'chip' +-- where hook %s +) AS R +USING (stage_id) +WHERE R.remote_id IS NULL + + + + + Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_stack.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_stack.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_stack.sql (revision 37068) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT stack_id AS stage_id + FROM stackRun +WHERE +stackRun.state = 'new' +-- where hook %s +GROUP BY stack_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING (remote_id) +WHERE +remoteRun.stage = 'stack' +-- where hook %s +) AS R +USING (stage_id) +WHERE R.remote_id IS NULL Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_warp.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_warp.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_definebyquery_warp.sql (revision 37068) @@ -0,0 +1,24 @@ +SELECT stage_id FROM + +( SELECT warp_id AS stage_id, count(warpSkyCellMap.skycell_id) AS N + FROM warpRun + LEFT JOIN warpSkyCellMap USING(warp_id) +WHERE +warpRun.state = 'new' +-- where hook %s +GROUP BY warp_id +HAVING N != 0 +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING (remote_id) +WHERE +remoteRun.stage = 'warp' +-- where hook %s +) AS R +USING (stage_id) +WHERE R.remote_id IS NULL + Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_dropcomponent.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_dropcomponent.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_dropcomponent.sql (revision 37068) @@ -0,0 +1,4 @@ +DELETE FROM remoteComponent +WHERE +remote_id = %ld AND +stage_id = %ld Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_updatepoll.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_updatepoll.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/remotetool_updatepoll.sql (revision 37068) @@ -0,0 +1,3 @@ +UPDATE remoteRun SET last_poll = CURRENT_TIMESTAMP +WHERE +%s Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/stacktool_addsummary.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/stacktool_addsummary.sql (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/stacktool_addsummary.sql (revision 37068) @@ -1,4 +1,4 @@ INSERT INTO stackSummary (sass_id,projection_cell,path_base) VALUES ( - %lld, -- warp_id + %lld, -- sass_id '%s', -- projection_cell '%s' -- path_base Index: branches/eam_branches/ipp-ops-20130712/ippTools/share/stacktool_revertsummary.sql =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/share/stacktool_revertsummary.sql (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/share/stacktool_revertsummary.sql (revision 37068) @@ -0,0 +1,3 @@ +DELETE FROM stackSummary + WHERE + sass_id = %lld Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/Makefile.am (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/Makefile.am (revision 37068) @@ -33,5 +33,7 @@ vptool \ sctool \ - releasetool + releasetool \ + fftool \ + remotetool pkginclude_HEADERS = \ @@ -87,5 +89,7 @@ vptool.h \ sctool.h \ - releasetool.h + releasetool.h \ + fftool.h \ + remotetool.h lib_LTLIBRARIES = libpxtools.la @@ -332,4 +336,16 @@ releasetoolConfig.c +fftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +fftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +fftool_SOURCES = \ + fftool.c \ + fftoolConfig.c + +remotetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +remotetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +remotetool_SOURCES = \ + remotetool.c \ + remotetoolConfig.c + clean-local: -rm -f TAGS Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/camtool.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/camtool.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/camtool.c (revision 37068) @@ -501,4 +501,6 @@ PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); PXOPT_LOOKUP_S16(background_model, config->args, "-background_model", false, false); + PXOPT_LOOKUP_S64(astrom_chips, config->args, "-astrom_chips", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); @@ -693,5 +695,5 @@ quality, background_model, - 0 + astrom_chips ); Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/camtoolConfig.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/camtoolConfig.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/camtoolConfig.c (revision 37068) @@ -205,4 +205,5 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_uq", 0, "define deteff_uq", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-background_model", 0, "set background_model value", 0); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-astrom_chips", 0, "chips with successful astrom", 0); // -processedexp Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/detselect.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/detselect.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/detselect.c (revision 37068) @@ -29,4 +29,6 @@ static bool searchMode(pxConfig *config); static bool selectMode(pxConfig *config); +static bool showMode(pxConfig *config); + # define MODECASE(caseName, func) \ @@ -50,4 +52,5 @@ MODECASE(DETSELECT_MODE_SEARCH, searchMode); MODECASE(DETSELECT_MODE_SELECT, selectMode); + MODECASE(DETSELECT_MODE_SHOW, showMode); default: psAbort("invalid option (this should not happen)"); @@ -241,2 +244,64 @@ return true; } + +static bool showMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_COPY_STR(config->args, where, "-type", "det_type", "=="); + + psString query = pxDataGet("detselect_show.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // we choose the single detrend image which matches all criteria and has + // the latest insertion date + + psStringAppend(&query, " ORDER BY registered DESC, iteration DESC"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detRun rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + return(true); +} Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/detselect.h =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/detselect.h (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/detselect.h (revision 37068) @@ -27,4 +27,5 @@ DETSELECT_MODE_SEARCH, DETSELECT_MODE_SELECT, + DETSELECT_MODE_SHOW } detselectMode; Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/detselectConfig.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/detselectConfig.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/detselectConfig.c (revision 37068) @@ -65,4 +65,12 @@ psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddBool(selectArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -show + psMetadata *showArgs = psMetadataAlloc(); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-det_type", 0, "search by detrend type", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-type", 0, "search by detrend type (alias for -det_type)", NULL); + psMetadataAddBool(showArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadata *argSets = psMetadataAlloc(); @@ -71,4 +79,5 @@ PXOPT_ADD_MODE("-search", "search for an appropriate detrend", DETSELECT_MODE_SEARCH, searchArgs); PXOPT_ADD_MODE("-select", "retreive detrend information", DETSELECT_MODE_SELECT, selectArgs); + PXOPT_ADD_MODE("-show", "show all active detrends of this type with constraints", DETSELECT_MODE_SHOW, showArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/disttool.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/disttool.c (revision 37068) @@ -413,4 +413,21 @@ magic = false; + } else if (!strcmp(stage, "ff")) { + runJoinStr = "fullForceRun.ff_id"; + query = pxDataGet("disttool_definebyquery_ff.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (fullForceRun.label = '%s') ", label); + } + if (dist_group) { + psStringAppend(&query, " AND (fullForceRun.dist_group = '%s') ", dist_group); + } + + magic = false; } else { psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); @@ -677,5 +694,5 @@ PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "distRun.data_group", "LIKE"); // require data_group or dist_id to be supplied @@ -759,5 +776,5 @@ query = "DELETE FROM distComponent where dist_id = %" PRId64; - char *query2 = "DELETE from rcDSFileset using distRun, rcDSFileset WHERE distRun.dist_id = rcDSFileset.dist_id AND rcDSFileset.state ='cleaned' AND dist_id = %" PRId64; + char *query2 = "DELETE from rcDSFileset using distRun, rcDSFileset WHERE distRun.dist_id = rcDSFileset.dist_id AND rcDSFileset.state ='cleaned' AND rcDSFileset.dist_id = %" PRId64; for (long i=0; i < psArrayLength(output); i++) { Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/disttoolConfig.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/disttoolConfig.c (revision 37068) @@ -91,5 +91,5 @@ // -startover psMetadata *startoverArgs = psMetadataAlloc(); - psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group", NULL); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group (LIKE comparison)", NULL); psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/fftool.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/fftool.c (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/fftool.c (revision 37068) @@ -0,0 +1,1103 @@ +/* + * fftool.c + * + * Copyright (C) 2013 Institute for Astronomy, University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxspace.h" +#include "fftool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool todoMode(pxConfig *config); +static bool addresultMode(pxConfig *config); +static bool resultMode(pxConfig *config); +static bool revertMode(pxConfig *config); +static bool updateresultMode(pxConfig *config); +static bool toadvanceMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); +static bool revertsummaryMode(pxConfig *config); +static bool updatesummaryMode(pxConfig *config); +static bool summaryMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); +static bool summaryMode(pxConfig *config); + +static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = fftoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(FFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(FFTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(FFTOOL_MODE_TODO, todoMode); + MODECASE(FFTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(FFTOOL_MODE_RESULT, resultMode); + MODECASE(FFTOOL_MODE_REVERT, revertMode); + MODECASE(FFTOOL_MODE_UPDATERESULT, updateresultMode); + MODECASE(FFTOOL_MODE_TOADVANCE, toadvanceMode); + MODECASE(FFTOOL_MODE_ADDSUMMARY, addsummaryMode); + MODECASE(FFTOOL_MODE_REVERTSUMMARY, revertsummaryMode); + MODECASE(FFTOOL_MODE_UPDATESUMMARY, updatesummaryMode); + MODECASE(FFTOOL_MODE_SUMMARY, summaryMode); + MODECASE(FFTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(FFTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + // optional + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + + PXOPT_LOOKUP_STR(sources_path_base, config->args, "-set_sources_path_base", false, false); + + psMetadata *skycalWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_label", "skycalRun.label", "LIKE"); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, skycalWhereMD, "-select_skycal_id", "skycalRun.skycal_id", "=="); + + if (!psListLength(skycalWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required"); + psFree(skycalWhereMD); + return false; + } + + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_filter", "stackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, skycalWhereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + psFree(skycalWhereMD); + return false; + } + + psMetadata *warpWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","warpRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "warpRun.warp_id", "=="); + if (!psListLength(warpWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required"); + psFree(warpWhereMD); + psFree(skycalWhereMD); + return false; + } + PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "warpRun.tess_id", "=="); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "rawExp.filter", "LIKE"); + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString select = pxDataGet("fftool_definebyquery.sql"); + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(skycalWhereMD); + psFree(warpWhereMD); + return false; + } + + psString where = NULL; + psString whereClause = psDBGenerateWhereConditionSQL(skycalWhereMD, NULL); + psStringAppend(&where, "\nAND %s", whereClause); + psStringAppend(&select, " %s", where); + psFree(whereClause); + psFree(skycalWhereMD); + + psString joinHook = NULL; + if (!rerun) { + psStringAppend(&joinHook, "\nLEFT JOIN fullForceRun ON fullForceRun.skycal_id = skycalRun.skycal_id"); + psStringAppend(&joinHook, "\n %s\nAND fullForceRun.label = '%s'", where, label); + psStringAppend(&select, "\nAND ff_id IS NULL"); + } + + if (!p_psDBRunQueryF(config->dbh, select, joinHook)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + return false; + } + psFree(select); + psFree(joinHook); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("fftool: no rows found"); + psFree(output); + psFree(where); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "toFullForce", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } + + psString warpQueryTemplate = pxDataGet("fftool_definebyquery_select_warps.sql"); + + whereClause = psDBGenerateWhereConditionSQL(warpWhereMD, NULL); + psStringAppend(&warpQueryTemplate, "\nAND %s", whereClause); + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); + psS64 skycal_id = psMetadataLookupS64(&status, row, "skycal_id"); + + psString path_base = NULL; + if (sources_path_base) { + path_base = sources_path_base; + } else { + path_base = psMetadataLookupStr(&status, row, "path_base"); + psAssert(status, "failed to find skycal path_base?"); + } + + psString skycal_data_group = psMetadataLookupStr(&status, row, "data_group"); + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + psString filter = psMetadataLookupStr(&status, row, "filter"); + + psString query = NULL; + psStringAppend(&query, warpQueryTemplate, tess_id, skycell_id, filter); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // Find the warps for this skycell and filter combination + psArray *warpOutput = p_psDBFetchResult(config->dbh); + if (!warpOutput) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(warpOutput)) { + // no warps for this skycal. Suprise? + psFree(warpOutput); + psFree(query); + continue; + } + + // create a staticskyRun + if (!fullForceRunInsert(config->dbh, + 0x0, // ff_id + skycal_id, + path_base, + "new", // state + workdir, + label, + data_group ? data_group : (skycal_data_group ? skycal_data_group : label), + dist_group, + note, + reduction, + NULL // registered + ) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + psS64 ff_id = psDBLastInsertID(config->dbh); + + for (int j = 0; j < warpOutput->n; j++) { + psMetadata *warpRow = warpOutput->data[j]; + psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id"); + + if (!fullForceInputInsert(config->dbh, + ff_id, + warp_id) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + psFree(warpOutput); + } + psFree(output); + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "fullForceRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE fullForceRun JOIN skycalRun USING(skycal_id) JOIN stackRun USING(stack_id) JOIN skycell USING(tess_id, skycell_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "fullForceRun", "ff_id", "fullForceResult", true, false); + psFree(query); + psFree(where); + + return result; +} + + +static bool todoMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "fullForceRun.label", "=="); + pxskycellAddWhere(config, whereMD); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_todo.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fullForceRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // required + PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", true, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(software_ver, config->args, "-software_ver", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + // XXX not sure we need a transaction here... + if (!fullForceResultInsert(config->dbh, + ff_id, + warp_id, + path_base, + dtime_script, + quality, + hostname, + software_ver, + fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool resultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceResult.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "fullForceRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceResult.fault", "=="); + PXOPT_COPY_S16(config->args, where, "-quality", "fullForceResult.quality", "=="); + pxskycellAddWhere(config, where); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_result.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "fullForceResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool revertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceResult.warp_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceResult.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("fftool_revert.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("fftool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} + +static bool updateresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + + if (!pxSetFaultCode(config->dbh, "fullForceResult", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} + +static bool setfullForceRunState(pxConfig *config, psS64 ff_id, const char *state) +{ + psString query = "UPDATE fullForceRun SET state = 'full' WHERE ff_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, ff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool toadvanceMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "fullForceRun.label", "=="); + pxskycellAddWhere(config, whereMD); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_toadvance.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = NULL; + psString temp = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&whereClause, "\n AND %s", temp); + psFree(temp); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + // the where clause is required and is added to the query by the "WHERE hook" format string + if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fullForceRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + // required + PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", true, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(software_ver, config->args, "-software_ver", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fullForceSummaryInsert(config->dbh, + ff_id, + path_base, + dtime_script, + quality, + hostname, + software_ver, + fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fault) { + if (!setfullForceRunState(config, ff_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change staticskyRun state"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool revertsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceSummary.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("fftool_revertsummary.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("fftool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} +static bool updatesummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + + + if (!pxSetFaultCode(config->dbh, "fullForceSummary", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} +static bool summaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceInput.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "fullForceRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + pxskycellAddWhere(config, where); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_summary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "fullForceSummary", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // PXOPT_LOOKUP_S64(det_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); +// PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + + ExportTable tables [] = { + {"fullForceRun", "fftool_export_run.sql"}, + {"fullForceInput", "fftool_export_input.sql"}, + {"fullForceResult", "fftool_export_result.sql"}, + {"fullForceSummary", "fftool_export_summary.sql"}, + }; + + int numTables = sizeof(tables)/sizeof(tables[0]); + + for (int i=0; i < numTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + int numImportTables = 3; + + char tables[3] [80] = {"fullForceInput", "fullForceResult", "fullForceSummary"}; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + +#ifdef notdef + fprintf (stderr, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + + psMetadataItem *item = psMetadataLookup (input, "fullForceRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceRunRow *fullForceRun = fullForceRunObjectFromMetadata (entry->data.md); + fullForceRunInsertObject (config->dbh, fullForceRun); + + // fprintf (stdout, "---- warp run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + for (int i = 0; i < numImportTables; i++) { + item = psMetadataLookup (input, tables[i]); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + switch (i) { + case 0: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceInputRow *fullForceInput = fullForceInputObjectFromMetadata (entry->data.md); + fullForceInputInsertObject (config->dbh, fullForceInput); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + + case 1: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceResultRow *fullForceResult = fullForceResultObjectFromMetadata (entry->data.md); + fullForceResultInsertObject (config->dbh, fullForceResult); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + + case 2: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceSummaryRow *fullForceSummary = fullForceSummaryObjectFromMetadata (entry->data.md); + fullForceSummaryInsertObject (config->dbh, fullForceSummary); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + } + } + return true; +} Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/fftool.h =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/fftool.h (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/fftool.h (revision 37068) @@ -0,0 +1,45 @@ +/* + * fftool.h + * + * Copyright (C) 2013 IfA Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FFTOOL_H +#define FFTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + FFTOOL_MODE_NONE = 0x0, + FFTOOL_MODE_DEFINEBYQUERY, + FFTOOL_MODE_UPDATERUN, + FFTOOL_MODE_TODO, + FFTOOL_MODE_ADDRESULT, + FFTOOL_MODE_RESULT, + FFTOOL_MODE_REVERT, + FFTOOL_MODE_UPDATERESULT, + FFTOOL_MODE_TOADVANCE, + FFTOOL_MODE_ADDSUMMARY, + FFTOOL_MODE_REVERTSUMMARY, + FFTOOL_MODE_UPDATESUMMARY, + FFTOOL_MODE_SUMMARY, + FFTOOL_MODE_EXPORTRUN, + FFTOOL_MODE_IMPORTRUN, +} fftoolMode; + +pxConfig *fftoolConfig(pxConfig *config, int argc, char **argv); + +#endif // FFTOOL_H Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/fftoolConfig.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/fftoolConfig.c (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/fftoolConfig.c (revision 37068) @@ -0,0 +1,245 @@ +/* + * fftool.c + * + * Copyright (C) 2013 Institute for Astronomy, University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "fftool.h" + +pxConfig *fftoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + (void) now; + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_label", PS_META_DUPLICATE_OK, "search by warp label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_label", PS_META_DUPLICATE_OK, "search by skycal label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_data_group", PS_META_DUPLICATE_OK, "search by warp data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_data_group", PS_META_DUPLICATE_OK, "search by skycal data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_id", 0, "search by skycal ID", 0); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "mimimum good_frac in warp", 0.0); + + pxskycellAddArguments(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_sources_path_base", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + pxskycellAddArguments(updaterunArgs); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + + // -todo + psMetadata *todoArgs = psMetadataAlloc(); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(todoArgs); + psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(todoArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresult + psMetadata *addresultArgs = psMetadataAlloc(); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -result + psMetadata *resultArgs= psMetadataAlloc(); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-quality", 0, "search by quality value", 0); + pxskycellAddArguments(resultArgs); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revert + psMetadata *revertArgs= psMetadataAlloc(); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertArgs); + + // -updateresult + psMetadata *updateresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality code", 0); + + // -toadvance + psMetadata *toadvanceArgs = psMetadataAlloc(); + psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(toadvanceArgs); + psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); + psMetadataAddF32(addsummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -updatesummary + psMetadata *updatesummaryArgs = psMetadataAlloc(); + psMetadataAddS64(updatesummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF32(updatesummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -summary + psMetadata *summaryArgs= psMetadataAlloc(); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + pxskycellAddArguments(summaryArgs); + psMetadataAddS16(summaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertsummary + psMetadata *revertsummaryArgs= psMetadataAlloc(); + psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(revertsummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertsummaryArgs); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ff_id", 0, "export this full force ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); +// psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "Define a new full force run", FFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "Update a run", FFTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-todo", "Get runs to do", FFTOOL_MODE_TODO, todoArgs); + PXOPT_ADD_MODE("-addresult", "Add result for fullforce run on a warp", FFTOOL_MODE_ADDRESULT, addresultArgs); + PXOPT_ADD_MODE("-result", "Get result fullforce run on a warp", FFTOOL_MODE_RESULT, resultArgs); + PXOPT_ADD_MODE("-revert", "Revert failed fullforce run on a warp", FFTOOL_MODE_REVERT, revertArgs); + PXOPT_ADD_MODE("-updateresult", "Update result for fullforce run on a warp", FFTOOL_MODE_UPDATERESULT, updateresultArgs); + PXOPT_ADD_MODE("-toadvance", "list completed runs to summarize", FFTOOL_MODE_TOADVANCE, toadvanceArgs); + PXOPT_ADD_MODE("-addsummary", "insert summary result", FFTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-updatesummary", "update summary result", FFTOOL_MODE_UPDATESUMMARY, updatesummaryArgs); + PXOPT_ADD_MODE("-revertsummary", "revert faulted summary", FFTOOL_MODE_REVERTSUMMARY, revertsummaryArgs); + PXOPT_ADD_MODE("-summary", "list summary results", FFTOOL_MODE_SUMMARY, summaryArgs); + PXOPT_ADD_MODE("-exportrun", "list summary results", FFTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "list summary results", FFTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/regtool.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/regtool.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/regtool.c (revision 37068) @@ -310,4 +310,9 @@ } + +// CZW: 2013-12-11 A bad exposure caused this code to fail to return any rows. +// I don't clearly see any logic flaws, and so I fixed the issue by +// retricting dateobs_begin to exclude this exposure. Problem for +// another day. static bool pendingburntoolimfileMode(pxConfig *config) { Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/regtoolConfig.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/regtoolConfig.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/regtoolConfig.c (revision 37068) @@ -292,5 +292,5 @@ psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-state", 0, "search by exposure state", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); - psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetool.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetool.c (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetool.c (revision 37068) @@ -0,0 +1,468 @@ +/* + * remotetool.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "remotetool.h" + +// Create +static bool definebyqueryMode(pxConfig *config); + +// List +static bool listrunMode(pxConfig *config); +static bool listcomponentMode(pxConfig *config); + +// Update +static bool dropcomponentMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool updatepollMode(pxConfig *config); +static bool revertrunMode(pxConfig *config); +static bool revertauthMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = remotetoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(REMOTETOOL_MODE_LISTRUN, listrunMode); + MODECASE(REMOTETOOL_MODE_LISTCOMPONENT, listcomponentMode); + MODECASE(REMOTETOOL_MODE_DROPCOMPONENT, dropcomponentMode); + MODECASE(REMOTETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(REMOTETOOL_MODE_UPDATEPOLL, updatepollMode); + MODECASE(REMOTETOOL_MODE_REVERTRUN, revertrunMode); + MODECASE(REMOTETOOL_MODE_REVERTAUTH, revertauthMode); + + default: + psAbort("invalid option (this should not happen)"); + } + psTrace("remotetool", 9, "Attempting to free config\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(EXIT_SUCCESS); + + FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +// Create + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // These are required + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // These are not + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S16(limit, config->args, "-limit", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + set_label = set_label ? set_label : label; + + // Get a list of the things we can insert into a new component + // CZW: these labels no longer need to be linked to the table, as I've split stageRun and remoteRun into sub-queries. + psString query = NULL; + psString whereOption = psStringCopy(""); + if (!strcmp(stage,"chip")) { + query = pxDataGet("remotetool_definebyquery_chip.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&whereOption, "\n AND (label = '%s')", label); + } + } + else if (!strcmp(stage,"camera")) { + query = pxDataGet("remotetool_definebyquery_cam.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&whereOption, "\n AND (label = '%s')", label); + } + } + else if (!strcmp(stage,"warp")) { + query = pxDataGet("remotetool_definebyquery_warp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + // Because warp has to check to see if something other than the state is set correctly, I have to use this wonky way to pass the label in. + // psStringAppend(&query, " AND (warpRun.label = '%s')", label); + psStringAppend(&whereOption, "\n AND (label = '%s')", label); + } + } + else if (!strcmp(stage,"stack")) { + query = pxDataGet("remotetool_definebyquery_stack.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&whereOption, "\n AND (label = '%s')", label); + } + } + else { + psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); + return false; + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQueryF(config->dbh, query,whereOption,whereOption)) { // This needs to be here twice because we need to restrict on label twice. + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("remotetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + if (!ippdbPrintMetadatas(stdout, output, "newremoteComponents", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + // Insert the top level run object + remoteRunRow *run = remoteRunRowAlloc(0, // remote_id + "new", // state + stage, + set_label, + path_base, + -1, // job_id + NULL, // last_poll + 0 // fault + ); + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object"); + return(true); + } + + if (!remoteRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return(true); + } + psS64 remote_id = psDBLastInsertID(config->dbh); + + psArray *list = psArrayAllocEmpty(limit); + for (long i=0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); + + remoteComponentRow *comp = remoteComponentRowAlloc(remote_id, + stage_id); + if (!comp) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object"); + return(true); + } + + if (!remoteComponentInsertObject(config->dbh, comp)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return(true); + } + + psArrayAdd(list, list->n, comp); + psFree(comp); + + } + psFree(run); + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!remoteComponentPrintObjects(stdout, list, !simple)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(list); + psFree(output); + + return true; +} + +// List +static bool listrunMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-poll_begin", "last_poll", ">="); + PXOPT_COPY_TIME(config->args, where, "-poll_end", "last_poll", "<="); + + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + PXOPT_LOOKUP_S16(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + + psArray *remRuns = NULL; + remRuns = psDBSelectRows(config->dbh, "remoteRun", where, limit); + + if (!ippdbPrintMetadatas(stdout, remRuns, "remoteRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(remRuns); + return false; + } + psFree(remRuns); + + return true; +} + +static bool listcomponentMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psArray *remComps = NULL; + remComps = psDBSelectRows(config->dbh, "remoteComponent", where, 0); + + if (!ippdbPrintMetadatas(stdout, remComps, "remoteComponent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(remComps); + return false; + } + psFree(remComps); + + return true; +} + +// Update + +static bool dropcomponentMode(pxConfig *config) +{ + // Assert that we have a unique component to operate on. + PXOPT_LOOKUP_S64(remote_id,config->args, "-remote_id",true, false); + psAssert(remote_id,"This must exist for this mode to be safe."); + PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); + psAssert(stage_id,"This must exist for this mode to be safe."); + + psString query = pxDataGet("remotetool_dropcomponent.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, remote_id, stage_id)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + return false; + } + psFree(query); + + + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + psAssert(remote_id,"This should have failed before this point."); + + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // Wheres + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + + // Values to set + PXOPT_COPY_STR(config->args, values, "-set_label", "label", "=="); + PXOPT_COPY_STR(config->args, values, "-set_state", "state", "=="); + PXOPT_COPY_S16(config->args, values, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, values, "-job_id", "job_id", "=="); + + long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); + psFree(values); + + if (rows < 1) { + psError(PS_ERR_UNKNOWN,false, "no rows affected"); + return(false); + } + + return(true); +} + +static bool updatepollMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + psAssert(remote_id,"This should have failed before this point."); + + psMetadata *where = psMetadataAlloc(); + // Wheres + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + + psString query = NULL; + query = pxDataGet("remotetool_updatepoll.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + psFree(whereClause); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query,whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(whereClause); + return false; + } + psFree(query); + psFree(whereClause); + return(true); +} + + + + +static bool revertrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // required + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + // Value to set + PXOPT_COPY_S16(config->args, values, "-fault", "fault", "=="); + long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); + psFree(values); + psFree(where); + if (rows < 1) { + psError(PS_ERR_UNKNOWN,false, "no rows affected"); + return(false); + } + + return(true); +} + +static bool revertauthMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // required + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psMetadataAddStr(where, PS_LIST_TAIL, "state", 0, "==", "auth"); + + // Value to set + psMetadataAddStr(values, PS_LIST_TAIL, "state", 0, "==", "run"); + + long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); + psFree(values); + psFree(where); + if (rows < 1) { + psError(PS_ERR_UNKNOWN,false, "no rows affected"); + return(false); + } + + return(true); +} + Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetool.h =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetool.h (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetool.h (revision 37068) @@ -0,0 +1,24 @@ +/* + * remotetool.h + */ + +#ifndef REMOTETOOL_H +#define REMOTETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + REMOTETOOL_MODE_NONE = 0x0, + REMOTETOOL_MODE_DEFINEBYQUERY, + REMOTETOOL_MODE_LISTRUN, + REMOTETOOL_MODE_LISTCOMPONENT, + REMOTETOOL_MODE_DROPCOMPONENT, + REMOTETOOL_MODE_UPDATERUN, + REMOTETOOL_MODE_UPDATEPOLL, + REMOTETOOL_MODE_REVERTRUN, + REMOTETOOL_MODE_REVERTAUTH, +} remotetoolMode; + +pxConfig *remotetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // REMOTETOOL_H Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetoolConfig.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetoolConfig.c (revision 37068) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/remotetoolConfig.c (revision 37068) @@ -0,0 +1,131 @@ +/* + * remotetoolConfig.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include "pxtools.h" +#include "remotetool.h" + +#define ADD_OPT(TYPE,TARG,NAME,COMMENT,DEFAULT) psMetadataAdd##TYPE(TARG, PS_LIST_TAIL, NAME, 0, COMMENT, DEFAULT) + +pxConfig *remotetoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + ADD_OPT(Str, definebyqueryArgs, "-label", "stage label to query", NULL); + ADD_OPT(Str, definebyqueryArgs, "-set_label", "remRun label to use", NULL); + ADD_OPT(Str, definebyqueryArgs, "-stage", "stage to query", NULL); + ADD_OPT(Str, definebyqueryArgs, "-path_base", "define base output location", NULL); + ADD_OPT(S16, definebyqueryArgs, "-limit", "limit to number of jobs to bundle",0); + ADD_OPT(Bool, definebyqueryArgs, "-pretend", "pretend to do this.", 0); + ADD_OPT(Bool, definebyqueryArgs,"-simple", "simple print format",0); + + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + ADD_OPT(S64, listrunArgs, "-remote_id", "run to list", 0); + ADD_OPT(Str, listrunArgs, "-state", "run state", NULL); + ADD_OPT(Str, listrunArgs, "-stage", "stage to return", NULL); + ADD_OPT(Str, listrunArgs, "-label", "remRun label to use", NULL); + + ADD_OPT(S64, listrunArgs, "-job_id", "job_id", 0); + // ADD_OPT(Str, listrunArgs, "-poll_begin", "last polled time begin", NULL); + // ADD_OPT(Str, listrunArgs, "-poll_end", "last polled time begin", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-poll_begin", 0, "last polled time begin", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-poll_end", 0, "last polled time end", NULL); + + ADD_OPT(S16, listrunArgs, "-fault", "fault code", 0); + + ADD_OPT(Bool, listrunArgs, "-simple", "simple print format", false); + ADD_OPT(S16, listrunArgs, "-limit", "limit to number of runs to display", 0); + + // -listcomponents + psMetadata *listcomponentArgs = psMetadataAlloc(); + ADD_OPT(S64, listcomponentArgs, "-remote_id", "run to list", 0); + ADD_OPT(S16, listcomponentArgs, "-limit", "limit to number of runs to display", 0); + ADD_OPT(Bool, listcomponentArgs, "-simple", "simple print format", false); + + // -dropcomponent + psMetadata *dropcomponentArgs = psMetadataAlloc(); + ADD_OPT(S64, dropcomponentArgs, "-remote_id", "run to update", 0); + ADD_OPT(S64, dropcomponentArgs, "-stage_id", "stage_id to remote", 0); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + ADD_OPT(S64, updaterunArgs, "-remote_id", "run to update", 0); + // ADD_OPT(Str, updaterunArgs, "-label", "label to update", NULL); + ADD_OPT(Str, updaterunArgs, "-set_label", "remRun label to use", NULL); + ADD_OPT(Str, updaterunArgs, "-set_state", "remoteRun state to assign", NULL); + ADD_OPT(S64, updaterunArgs, "-job_id", "job_id to set", 0); + ADD_OPT(S16, updaterunArgs, "-fault", "fault code", 0); + // ADD_OPT(Bool, updaterunArgs, "-poll", "set poll date", NULL); + + // -updatepoll + psMetadata *updatepollArgs = psMetadataAlloc(); + ADD_OPT(S64, updatepollArgs, "-remote_id", "run to update", 0); + + // -revertrun + psMetadata *revertrunArgs = psMetadataAlloc(); + ADD_OPT(S64, revertrunArgs, "-remote_id", "run to revert", 0); + ADD_OPT(Str, revertrunArgs, "-label", "remRun label to use", NULL); + ADD_OPT(S16, revertrunArgs, "-fault", "fault code", 0); + + // -revertauth + psMetadata *revertauthArgs = psMetadataAlloc(); + ADD_OPT(S64, revertauthArgs, "-remote_id", "run to revert", 0); + ADD_OPT(Str, revertauthArgs, "-label", "remRun label to use", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "", REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-listrun", "", REMOTETOOL_MODE_LISTRUN, listrunArgs); + PXOPT_ADD_MODE("-listcomponent", "", REMOTETOOL_MODE_LISTCOMPONENT, listcomponentArgs); + PXOPT_ADD_MODE("-dropcomponent", "", REMOTETOOL_MODE_DROPCOMPONENT, dropcomponentArgs); + PXOPT_ADD_MODE("-updaterun", "", REMOTETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-updatepoll", "", REMOTETOOL_MODE_UPDATEPOLL, updatepollArgs); + PXOPT_ADD_MODE("-revertrun", "", REMOTETOOL_MODE_REVERTRUN, revertrunArgs); + PXOPT_ADD_MODE("-revertauth", "", REMOTETOOL_MODE_REVERTAUTH, revertauthArgs); + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PXTOOLS_ERR_SYS, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} + Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktool.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktool.c (revision 37068) @@ -44,4 +44,5 @@ static bool tosummaryMode(pxConfig *config); static bool addsummaryMode(pxConfig *config); +static bool revertsummaryMode(pxConfig *config); static bool summaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); @@ -85,4 +86,5 @@ MODECASE(STACKTOOL_MODE_TOSUMMARY, tosummaryMode); MODECASE(STACKTOOL_MODE_ADDSUMMARY, addsummaryMode); + MODECASE(STACKTOOL_MODE_REVERTSUMMARY, revertsummaryMode); MODECASE(STACKTOOL_MODE_SUMMARY, summaryMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); @@ -1570,4 +1572,32 @@ } +static bool revertsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + PXOPT_LOOKUP_S64(sass_id, config->args, "-sass_id", true, false); + + psString query = pxDataGet("stacktool_revertsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, sass_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return(false); + } + psLogMsg("stacktool", PS_LOG_INFO, "Deleted %ld rows", numUpdated); + + psFree(query); + + return(true); +} + static bool summaryMode(pxConfig *config) { Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktool.h =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktool.h (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktool.h (revision 37068) @@ -38,4 +38,5 @@ STACKTOOL_MODE_TOSUMMARY, STACKTOOL_MODE_ADDSUMMARY, + STACKTOOL_MODE_REVERTSUMMARY, STACKTOOL_MODE_SUMMARY, STACKTOOL_MODE_PENDINGCLEANUPRUN, Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktoolConfig.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/stacktoolConfig.c (revision 37068) @@ -265,4 +265,8 @@ psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -revertsummary + psMetadata *revertsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association ID", 0); + // -summary psMetadata *summaryArgs = psMetadataAlloc(); @@ -335,13 +339,18 @@ PXOPT_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs); PXOPT_ADD_MODE("-sumskyfile", "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); - PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); + PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileArgs); + PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", STACKTOOL_MODE_DONECLEANUP, donecleanupArgs); - PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + + PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs); PXOPT_ADD_MODE("-summary", "show runs that have been summarized", STACKTOOL_MODE_SUMMARY, summaryArgs); PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", STACKTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-revertsummary", "revert entry in the summary table", STACKTOOL_MODE_REVERTSUMMARY, revertsummaryArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", STACKTOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/eam_branches/ipp-ops-20130712/ippTools/src/staticskytool.c =================================================================== --- branches/eam_branches/ipp-ops-20130712/ippTools/src/staticskytool.c (revision 37064) +++ branches/eam_branches/ipp-ops-20130712/ippTools/src/staticskytool.c (revision 37068) @@ -1045,8 +1045,8 @@ PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); pxAddLabelSearchArgs(config, whereMD, "-select_label", "staticskyRun.label", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "staticskyRun.data_group", "LIKE"); - pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); if (!pxskycellAddWhere(config, whereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");