Index: branches/pap/ippTools/share/Makefile.am =================================================================== --- branches/pap/ippTools/share/Makefile.am (revision 27708) +++ branches/pap/ippTools/share/Makefile.am (revision 28003) @@ -31,4 +31,5 @@ chiptool_donecleanup.sql \ chiptool_find_rawexp.sql \ + chiptool_listrun.sql \ chiptool_pendingcleanupimfile.sql \ chiptool_pendingcleanuprun.sql \ @@ -36,10 +37,11 @@ chiptool_processedimfile.sql \ chiptool_revertprocessedimfile.sql \ + chiptool_revertupdatedimfile.sql \ chiptool_run.sql \ chiptool_runstate.sql \ chiptool_setimfiletoupdate.sql \ + chiptool_export_run.sql \ chiptool_export_imfile.sql \ chiptool_export_processed_imfile.sql \ - chiptool_export_run.sql \ chiptool_unmasked.sql \ detselect_search.sql \ @@ -109,8 +111,10 @@ difftool_export_skyfile.sql \ difftool_inputskyfile.sql \ + difftool_listrun.sql \ difftool_pendingcleanuprun.sql \ difftool_pendingcleanupskyfile.sql \ difftool_revertdiffskyfile_delete.sql \ - difftool_revertdiffskyfile_update.sql \ + difftool_revertdiffskyfile_updated.sql \ + difftool_setskyfiletoupdate.sql \ difftool_skyfile.sql \ difftool_todiffskyfile.sql \ @@ -194,4 +198,5 @@ magictool_rawimfile.sql \ magictool_revertnode.sql \ + magictool_exposure.sql \ magicdstool_clearstatefaults.sql \ magicdstool_completed_runs.sql \ @@ -221,12 +226,16 @@ pstamptool_completedreq.sql \ pstamptool_datastore.sql \ + pstamptool_getdependent.sql \ pstamptool_listjob.sql \ + pstamptool_pendingcleanup.sql \ pstamptool_pendingdependent.sql \ pstamptool_pendingjob.sql \ pstamptool_pendingreq.sql \ pstamptool_project.sql \ + pstamptool_revertdependent.sql \ pstamptool_revertjob.sql \ pstamptool_revertreq.sql \ pstamptool_revertreq_deletejobs.sql \ + pstamptool_updatejob.sql \ pxadmin_create_tables.sql \ pxadmin_create_mirror_tables.sql \ @@ -290,10 +299,13 @@ warptool_finish_run.sql \ warptool_imfile.sql \ + warptool_listrun.sql \ warptool_pendingcleanuprun.sql \ warptool_pendingcleanupskyfile.sql \ warptool_revertoverlap.sql \ warptool_revertwarped_delete.sql \ + warptool_revertwarped_updated.sql \ warptool_runstate.sql \ warptool_scmap.sql \ + warptool_setskyfiletoupdate.sql \ warptool_tooverlap.sql \ warptool_towarped.sql \ Index: branches/pap/ippTools/share/camtool_find_pendingexp.sql =================================================================== --- branches/pap/ippTools/share/camtool_find_pendingexp.sql (revision 27708) +++ branches/pap/ippTools/share/camtool_find_pendingexp.sql (revision 28003) @@ -10,5 +10,6 @@ rawExp.telescope, rawExp.filelevel, - chipRun.magicked AS chip_magicked + chipRun.magicked AS chip_magicked, + camProcessedExp.path_base FROM camRun JOIN chipRun Index: branches/pap/ippTools/share/camtool_find_processedexp.sql =================================================================== --- branches/pap/ippTools/share/camtool_find_processedexp.sql (revision 27708) +++ branches/pap/ippTools/share/camtool_find_processedexp.sql (revision 28003) @@ -3,6 +3,8 @@ camRun.workdir, camRun.label, + camRun.state, camRun.magicked, chipRun.chip_id, + chipRun.state as chip_state, chipRun.magicked as chip_magicked, rawExp.exp_tag, Index: branches/pap/ippTools/share/chiptool_change_imfile_data_state.sql =================================================================== --- branches/pap/ippTools/share/chiptool_change_imfile_data_state.sql (revision 27708) +++ branches/pap/ippTools/share/chiptool_change_imfile_data_state.sql (revision 28003) @@ -3,4 +3,5 @@ -- chip_id and class_id UPDATE chipProcessedImfile +JOIN chipRun USING(chip_id, exp_id) JOIN rawImfile USING(exp_id, class_id) SET Index: branches/pap/ippTools/share/chiptool_listrun.sql =================================================================== --- branches/pap/ippTools/share/chiptool_listrun.sql (revision 28003) +++ branches/pap/ippTools/share/chiptool_listrun.sql (revision 28003) @@ -0,0 +1,30 @@ +SELECT DISTINCT + chipRun.chip_id, + chipRun.state, + chipRun.workdir, + chipRun.label, + chipRun.data_group, + chipRun.dist_group, + chipRun.note, + chipRun.magicked, + chipRun.reduction, + camRun.cam_id, + camRun.state AS camRun_state, + camProcessedExp.path_base AS cam_path_base, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawExp.magicked AS raw_magicked, + magicDSRun.state AS dsRun_state +FROM chipRun +JOIN rawExp + USING(exp_id) +LEFT JOIN camRun USING(chip_id) +LEFT JOIN camProcessedExp USING(cam_id) +LEFT JOIN magicDSRun + ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place Index: branches/pap/ippTools/share/chiptool_pendingcleanupimfile.sql =================================================================== --- branches/pap/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 27708) +++ branches/pap/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 28003) @@ -13,7 +13,8 @@ USING(chip_id) WHERE - ((chipRun.state = 'goto_cleaned' AND chipProcessedImfile.data_state = 'full') + ((chipRun.state = 'goto_cleaned' AND (chipProcessedImfile.data_state = 'full' + OR chipProcessedImfile.data_state = 'update')) OR - (chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state = 'full') + (chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state != 'scrubbed') OR (chipRun.state = 'goto_purged' AND chipProcessedImfile.data_state != 'purged')) Index: branches/pap/ippTools/share/chiptool_pendingimfile.sql =================================================================== --- branches/pap/ippTools/share/chiptool_pendingimfile.sql (revision 27708) +++ branches/pap/ippTools/share/chiptool_pendingimfile.sql (revision 28003) @@ -10,5 +10,7 @@ rawExp.camera, rawExp.telescope, - rawExp.filelevel + rawExp.filelevel, + IFNULL(Label.priority, 10000) AS priority, + chipProcessedImfile.path_base FROM chipRun JOIN rawExp @@ -25,4 +27,5 @@ LEFT JOIN chipMask ON chipRun.label = chipMask.label +LEFT JOIN Label ON chipRun.label = Label.label WHERE ((chipRun.state = 'new' @@ -33,9 +36,5 @@ OR (chipRun.state = 'update' - AND chipProcessedImfile.data_state = 'update')) --- // Restriction to prevent the processing of bad chips. Not needed now. --- AND --- (rawImfile.class_id != 'XY15' AND --- rawImfile.class_id != 'XY32' AND --- rawImfile.class_id != 'XY45') - + AND chipProcessedImfile.data_state = 'update' + AND chipProcessedImfile.fault = 0)) + AND (Label.active OR Label.active IS NULL) Index: branches/pap/ippTools/share/chiptool_processedimfile.sql =================================================================== --- branches/pap/ippTools/share/chiptool_processedimfile.sql (revision 27708) +++ branches/pap/ippTools/share/chiptool_processedimfile.sql (revision 28003) @@ -22,5 +22,12 @@ rawExp.camera, rawExp.telescope, - rawExp.filelevel + rawExp.filelevel, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time, + rawImfile.magicked AS raw_magicked, + rawImfile.burntool_state, + magicDSRun.state AS dsRun_state FROM chipRun JOIN chipImfile @@ -30,2 +37,7 @@ JOIN rawExp ON chipProcessedImfile.exp_id = rawExp.exp_id +JOIN rawImfile + ON rawExp.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id +LEFT JOIN magicDSRun + ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place Index: branches/pap/ippTools/share/chiptool_revertupdatedimfile.sql =================================================================== --- branches/pap/ippTools/share/chiptool_revertupdatedimfile.sql (revision 28003) +++ branches/pap/ippTools/share/chiptool_revertupdatedimfile.sql (revision 28003) @@ -0,0 +1,8 @@ +UPDATE chipProcessedImfile + JOIN chipRun using(chip_id,exp_id) + JOIN rawExp using(exp_id) +SET chipProcessedImfile.fault = 0 +WHERE + chipRun.state = 'update' + AND chipProcessedImfile.data_state = 'update' + AND chipProcessedImfile.fault != 0 Index: branches/pap/ippTools/share/detselect_search.sql =================================================================== --- branches/pap/ippTools/share/detselect_search.sql (revision 27708) +++ branches/pap/ippTools/share/detselect_search.sql (revision 28003) @@ -27,3 +27,3 @@ ) as Foo WHERE - (state = 'stop' OR state = 'register') + (state = 'stop') Index: branches/pap/ippTools/share/difftool_definewarpstack_part1.sql =================================================================== --- branches/pap/ippTools/share/difftool_definewarpstack_part1.sql (revision 27708) +++ branches/pap/ippTools/share/difftool_definewarpstack_part1.sql (revision 28003) @@ -5,4 +5,5 @@ warpRun.label, warpRun.tess_id, + warpRun.data_group, COUNT(skycell_id) as skycell_count FROM warpRun Index: branches/pap/ippTools/share/difftool_definewarpwarp_select.sql =================================================================== --- branches/pap/ippTools/share/difftool_definewarpwarp_select.sql (revision 27708) +++ branches/pap/ippTools/share/difftool_definewarpwarp_select.sql (revision 28003) @@ -3,4 +3,5 @@ inputWarpRun.tess_id AS tess_id, inputRawExp.exp_id AS input_exp_id, + inputWarpRun.data_group AS input_data_group, -- The following trick pulls out the warp_id that has the smallest distance SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id Index: branches/pap/ippTools/share/difftool_listrun.sql =================================================================== --- branches/pap/ippTools/share/difftool_listrun.sql (revision 28003) +++ branches/pap/ippTools/share/difftool_listrun.sql (revision 28003) @@ -0,0 +1,66 @@ +SELECT DISTINCT + diffRun.diff_id, + diffRun.state, + diffRun.workdir, + diffRun.label, + diffRun.data_group, + diffRun.dist_group, + diffRun.note, + diffRun.magicked, + diffRun.diff_mode, + diffRun.tess_id, + diffRun.bothways, + warp1, + stack1, + warp2, + stack2, + -- The following are only valid for warps + -- XXX This needs to be more clever to handle diffs between stacks + -- Zero points are appropriate for both forward and backward diffs + camProcessedInput.zpt_obs, + camProcessedInput.zpt_stdev, + camProcessedInput.zpt_lq, + camProcessedInput.zpt_uq, + rawInput.comment, + rawInput.exp_time, + rawInput.camera, + rawInput.exp_name AS exp_name_1, + rawInput.exp_id AS exp_id_1, + chipInput.chip_id AS chip_id_1, + camInput.cam_id AS cam_id_1, + fakeInput.fake_id AS fake_id_1, + camProcessedInput.sigma_ra AS sigma_ra_1, + camProcessedInput.sigma_dec AS sigma_dec_1, + rawTemplate.exp_name AS exp_name_2, + rawTemplate.exp_id AS exp_id_2, + chipTemplate.chip_id AS chip_id_2, + camTemplate.cam_id AS cam_id_2, + fakeTemplate.fake_id AS fake_id_2, + camProcessedTemplate.sigma_ra AS sigma_ra_2, + camProcessedTemplate.sigma_dec AS sigma_dec_2 +FROM diffRun +JOIN diffInputSkyfile USING(diff_id) +LEFT JOIN warpRun AS warpInput + ON warpInput.warp_id = diffInputSkyfile.warp1 +LEFT JOIN fakeRun AS fakeInput + ON fakeInput.fake_id = warpInput.fake_id +LEFT JOIN camRun AS camInput + ON camInput.cam_id = fakeInput.cam_id +LEFT JOIN camProcessedExp AS camProcessedInput + ON camProcessedInput.cam_id = camInput.cam_id +LEFT JOIN chipRun AS chipInput + ON chipInput.chip_id = camInput.chip_id +LEFT JOIN rawExp AS rawInput + ON rawInput.exp_id = chipInput.exp_id +LEFT JOIN warpRun AS warpTemplate + ON warpTemplate.warp_id = diffInputSkyfile.warp2 +LEFT JOIN fakeRun AS fakeTemplate + ON fakeTemplate.fake_id = warpTemplate.fake_id +LEFT JOIN camRun AS camTemplate + ON camTemplate.cam_id = fakeTemplate.cam_id +LEFT JOIN camProcessedExp AS camProcessedTemplate + ON camProcessedTemplate.cam_id = camTemplate.cam_id +LEFT JOIN chipRun AS chipTemplate + ON chipTemplate.chip_id = camTemplate.chip_id +LEFT JOIN rawExp AS rawTemplate + ON rawTemplate.exp_id = chipTemplate.exp_id Index: branches/pap/ippTools/share/difftool_revertdiffskyfile_updated.sql =================================================================== --- branches/pap/ippTools/share/difftool_revertdiffskyfile_updated.sql (revision 28003) +++ branches/pap/ippTools/share/difftool_revertdiffskyfile_updated.sql (revision 28003) @@ -0,0 +1,6 @@ +UPDATE diffSkyfile + JOIN diffRun USING(diff_id) +SET diffSkyfile.fault = 0 +WHERE diffRun.state = 'update' + AND diffSkyfile.data_state = 'update' + AND diffSkyfile.fault != 0 Index: branches/pap/ippTools/share/difftool_setskyfiletoupdate.sql =================================================================== --- branches/pap/ippTools/share/difftool_setskyfiletoupdate.sql (revision 28003) +++ branches/pap/ippTools/share/difftool_setskyfiletoupdate.sql (revision 28003) @@ -0,0 +1,8 @@ +UPDATE diffRun + JOIN diffSkyfile USING(diff_id) +SET diffRun.state = 'update', + diffSkyfile.data_state = 'update' + -- set hook %s +WHERE diff_id = %lld + AND (diffRun.state = 'cleaned' OR diffRun.state = 'update') + AND (diffSkyfile.data_state = 'cleaned') Index: branches/pap/ippTools/share/difftool_skyfile.sql =================================================================== --- branches/pap/ippTools/share/difftool_skyfile.sql (revision 27708) +++ branches/pap/ippTools/share/difftool_skyfile.sql (revision 28003) @@ -7,4 +7,5 @@ diffRun.label, diffRun.bothways, + diffRun.diff_mode, warp1, stack1, Index: branches/pap/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- branches/pap/ippTools/share/difftool_todiffskyfile.sql (revision 27708) +++ branches/pap/ippTools/share/difftool_todiffskyfile.sql (revision 28003) @@ -12,5 +12,6 @@ diffRun.reduction, diffRun.bothways, - diffRun.diff_mode + diffRun.diff_mode, + diffSkyfile.path_base FROM diffRun JOIN diffInputSkyfile USING(diff_id) Index: branches/pap/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_todestreak_camera.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_todestreak_camera.sql (revision 28003) @@ -3,4 +3,5 @@ magicDSRun.magic_id, chipRun.exp_id, + camRun.magicked, magicDSRun.label, camera, Index: branches/pap/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_todestreak_chip.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_todestreak_chip.sql (revision 28003) @@ -3,4 +3,5 @@ magicDSRun.magic_id, chipRun.exp_id, + chipRun.magicked, magicDSRun.label, camera, @@ -33,5 +34,5 @@ magicDSRun.state = 'new' AND magicDSRun.stage = 'chip' - AND chipRun.state = 'full' + AND (chipRun.state = 'full' OR (chipRun.state = 'update' and chipProcessedImfile.data_state = 'full')) AND chipProcessedImfile.fault = 0 AND chipProcessedImfile.quality = 0 Index: branches/pap/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_todestreak_diff.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_todestreak_diff.sql (revision 28003) @@ -10,4 +10,5 @@ stage, magicRun.diff_id AS stage_id, + diffRun.magicked, diffSkyfile.skycell_id AS component, CAST(NULL AS CHAR(255)) AS uri, @@ -47,13 +48,14 @@ magicDSRun.label, rawExp.camera, - magicMask.uri as streaks_uri, + magicMask.uri AS streaks_uri, (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, stage, - magicRun.diff_id as stage_id, - diffSkyfile.skycell_id as component, + magicRun.diff_id AS stage_id, + diffRun.magicked, + diffSkyfile.skycell_id AS component, CAST(NULL AS CHAR(255)) AS uri, diffSkyfile.path_base, magicRun.inverse, - CAST(NULL AS CHAR(255)) as cam_path_base, + CAST(NULL AS CHAR(255)) AS cam_path_base, CAST(NULL AS CHAR(255)) AS cam_reduction, outroot, @@ -80,5 +82,5 @@ AND diffSkyfile.quality = 0 AND magicDSFile.component IS NULL -) as magicDSRun +) AS magicDSRun -- we need the following so this query is compatible with the other stages WHERE magic_ds_id IS NOT NULL Index: branches/pap/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_todestreak_raw.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_todestreak_raw.sql (revision 28003) @@ -4,9 +4,10 @@ magicRun.exp_id, magicDSRun.label, - camera, + rawExp.camera, magicMask.uri as streaks_uri, CAST(NULL AS CHAR(255)) AS inv_streaks_uri, stage, stage_id, + rawExp.magicked, class_id as component, rawImfile.uri AS uri, @@ -25,5 +26,6 @@ JOIN camRun USING(cam_id) JOIN camProcessedExp USING(cam_id) -JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id +JOIN rawExp ON magicRun.exp_id = rawExp.exp_id +JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id LEFT JOIN magicDSFile ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id Index: branches/pap/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_todestreak_warp.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_todestreak_warp.sql (revision 28003) @@ -9,4 +9,5 @@ stage, stage_id, + warpRun.magicked, warpSkyfile.skycell_id as component, warpSkyfile.uri, Index: branches/pap/ippTools/share/magicdstool_torevert_camera.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_torevert_camera.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_torevert_camera.sql (revision 28003) @@ -13,5 +13,6 @@ 0 AS bothways, 0 AS bytes, - 0 AS md5sum + 0 AS md5sum, + camRun.magicked FROM magicDSRun JOIN magicDSFile using(magic_ds_id) Index: branches/pap/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_torevert_chip.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_torevert_chip.sql (revision 28003) @@ -13,9 +13,12 @@ 0 AS bothways, 0 AS bytes, - 0 AS md5sum + 0 AS md5sum, + chipRun.magicked FROM magicDSRun JOIN magicDSFile using(magic_ds_id) JOIN camProcessedExp using(cam_id) - JOIN chipProcessedImfile ON (stage_id = chipProcessedImfile.chip_id AND component = class_id) + JOIN chipRun ON (stage_id = chipRun.chip_id) + -- JOIN chipProcessedImfile ON (stage_id = chipProcessedImfile.chip_id AND component = class_id) + JOIN chipProcessedImfile USING(chip_id, exp_id) JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'chip' Index: branches/pap/ippTools/share/magicdstool_torevert_diff.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_torevert_diff.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_torevert_diff.sql (revision 28003) @@ -13,5 +13,6 @@ CAST(diffRun.bothways AS SIGNED) AS bothways, 0 AS bytes, - 0 AS md5sum + 0 AS md5sum, + diffRun.magicked FROM magicDSRun JOIN magicRun using(magic_id) Index: branches/pap/ippTools/share/magicdstool_torevert_raw.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_torevert_raw.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_torevert_raw.sql (revision 28003) @@ -10,8 +10,9 @@ outroot, rawImfile.uri AS path_base, - NULL AS cam_path_base, + CAST(NULL AS CHAR(255)) AS cam_path_base, 0 AS bothways, bytes, - md5sum + md5sum, + 0 AS magicked FROM magicDSRun JOIN magicDSFile using(magic_ds_id) Index: branches/pap/ippTools/share/magicdstool_torevert_warp.sql =================================================================== --- branches/pap/ippTools/share/magicdstool_torevert_warp.sql (revision 27708) +++ branches/pap/ippTools/share/magicdstool_torevert_warp.sql (revision 28003) @@ -13,5 +13,6 @@ 0 AS bothways, 0 AS bytes, - 0 AS md5sum + 0 AS md5sum, + warpRun.magicked FROM magicDSRun JOIN magicDSFile USING(magic_ds_id) Index: branches/pap/ippTools/share/magictool_exposure.sql =================================================================== --- branches/pap/ippTools/share/magictool_exposure.sql (revision 28003) +++ branches/pap/ippTools/share/magictool_exposure.sql (revision 28003) @@ -0,0 +1,54 @@ +SELECT * +FROM ( + -- Forward diff + SELECT DISTINCT + magicRun.magic_id, + rawExp.*, + camProcessedExp.path_base, + camProcessedExp.zpt_obs, + camProcessedExp.zpt_stdev, + camProcessedExp.fwhm_major, + camProcessedExp.fwhm_minor, + camProcessedExp.iq_m2, + camProcessedExp.iq_m3, + camProcessedExp.iq_m4 + FROM + magicRun + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + LEFT JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 + AND magicRun.inverse = 0 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN camProcessedExp USING(cam_id) + JOIN chipRun USING(chip_id,exp_id) + JOIN rawExp USING(exp_id) + -- WHERE hook %s + UNION + -- Backward diff + SELECT DISTINCT + magicRun.magic_id, + rawExp.*, + camProcessedExp.path_base, + camProcessedExp.zpt_obs, + camProcessedExp.zpt_stdev, + camProcessedExp.fwhm_major, + camProcessedExp.fwhm_minor, + camProcessedExp.iq_m2, + camProcessedExp.iq_m3, + camProcessedExp.iq_m4 + FROM + magicRun + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + LEFT JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp2 + AND magicRun.inverse = 1 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN camProcessedExp USING(cam_id) + JOIN chipRun USING(chip_id,exp_id) + JOIN rawExp USING(exp_id) + -- WHERE hook %s +) AS magicExposures Index: branches/pap/ippTools/share/pstamptool_completedreq.sql =================================================================== --- branches/pap/ippTools/share/pstamptool_completedreq.sql (revision 27708) +++ branches/pap/ippTools/share/pstamptool_completedreq.sql (revision 28003) @@ -1,4 +1,6 @@ -SELECT * +SELECT pstampRequest.* , + IFNULL(Label.priority, 0) AS priority FROM pstampRequest + LEFT JOIN Label USING(label) WHERE state = 'run' AND ( Index: branches/pap/ippTools/share/pstamptool_getdependent.sql =================================================================== --- branches/pap/ippTools/share/pstamptool_getdependent.sql (revision 28003) +++ branches/pap/ippTools/share/pstamptool_getdependent.sql (revision 28003) @@ -0,0 +1,5 @@ +SELECT DISTINCT pstampDependent.* +FROM pstampDependent +-- JOIN pstampJob USING(dep_id) +-- JOIN pstampRequest USING(req_id) +WHERE pstampDependent.state = 'new' Index: branches/pap/ippTools/share/pstamptool_pendingcleanup.sql =================================================================== --- branches/pap/ippTools/share/pstamptool_pendingcleanup.sql (revision 28003) +++ branches/pap/ippTools/share/pstamptool_pendingcleanup.sql (revision 28003) @@ -0,0 +1,9 @@ +-- postage stamp requests pending cleanup +SELECT + pstampRequest.*, + IFNULL(Label.priority, 10000) AS priority +FROM pstampRequest + LEFT JOIN Label ON pstampRequest.label = Label.label +WHERE pstampRequest.state = 'goto_cleaned' + AND pstampRequest.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: branches/pap/ippTools/share/pstamptool_pendingdependent.sql =================================================================== --- branches/pap/ippTools/share/pstamptool_pendingdependent.sql (revision 27708) +++ branches/pap/ippTools/share/pstamptool_pendingdependent.sql (revision 28003) @@ -1,3 +1,12 @@ -SELECT * +SELECT DISTINCT + pstampDependent.*, + IFNULL(Label.priority, 10000) AS priority FROM pstampDependent -WHERE state = 'new' +JOIN pstampJob USING(dep_id) +JOIN pstampRequest USING(req_id) +LEFT JOIN Label ON pstampRequest.label = Label.label +WHERE pstampDependent.state = 'new' + AND pstampDependent.fault = 0 + AND pstampJob.state = 'run' + AND (pstampRequest.state = 'run' OR pstampRequest.state = 'new') + AND (Label.active OR Label.active IS NULL) Index: branches/pap/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- branches/pap/ippTools/share/pstamptool_pendingjob.sql (revision 27708) +++ branches/pap/ippTools/share/pstamptool_pendingjob.sql (revision 28003) @@ -1,6 +1,8 @@ -SELECT pstampJob.* +SELECT pstampJob.*, + IFNULL(Label.priority, 10000) AS priority FROM pstampJob JOIN pstampRequest USING(req_id) LEFT JOIN pstampDependent USING(dep_id) + LEFT JOIN Label ON pstampRequest.label = Label.label WHERE pstampRequest.state = 'run' AND pstampRequest.fault = 0 @@ -8,2 +10,3 @@ AND pstampJob.fault = 0 AND (dep_id = 0 OR pstampDependent.state = 'full') + AND (Label.active OR Label.active IS NULL) Index: branches/pap/ippTools/share/pstamptool_pendingreq.sql =================================================================== --- branches/pap/ippTools/share/pstamptool_pendingreq.sql (revision 27708) +++ branches/pap/ippTools/share/pstamptool_pendingreq.sql (revision 28003) @@ -1,9 +1,11 @@ -- postage stamp requests ready for download and parsing SELECT - pstampRequest.*, - pstampDataStore.outProduct as ds_outProduct - FROM pstampRequest - LEFT JOIN pstampDataStore - USING(ds_id) - WHERE pstampRequest.state = 'new' + pstampRequest.*, + pstampDataStore.outProduct AS ds_outProduct, + IFNULL(Label.priority, 10000) AS priority +FROM pstampRequest + LEFT JOIN pstampDataStore USING(ds_id) + LEFT JOIN Label ON pstampRequest.label = Label.label +WHERE pstampRequest.state = 'new' AND pstampRequest.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: branches/pap/ippTools/share/pstamptool_revertdependent.sql =================================================================== --- branches/pap/ippTools/share/pstamptool_revertdependent.sql (revision 28003) +++ branches/pap/ippTools/share/pstamptool_revertdependent.sql (revision 28003) @@ -0,0 +1,5 @@ +UPDATE pstampDependent + JOIN pstampJob USING(dep_id) + JOIN pstampRequest USING(req_id) +SET pstampDependent.fault = 0 +WHERE pstampDependent.state = 'new' Index: branches/pap/ippTools/share/pstamptool_updatejob.sql =================================================================== --- branches/pap/ippTools/share/pstamptool_updatejob.sql (revision 28003) +++ branches/pap/ippTools/share/pstamptool_updatejob.sql (revision 28003) @@ -0,0 +1,2 @@ +UPDATE pstampJob LEFT JOIN pstampDependent USING(dep_id) +SET Index: branches/pap/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/pap/ippTools/share/pxadmin_create_tables.sql (revision 27708) +++ branches/pap/ippTools/share/pxadmin_create_tables.sql (revision 28003) @@ -1175,4 +1175,6 @@ KEY(label), KEY(fault), + KEY(stage), + KEY(stage_id), FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id) @@ -1301,4 +1303,5 @@ outProduct VARCHAR(64), uri VARCHAR(255), + outdir VARCHAR(255), fault SMALLINT, PRIMARY KEY(req_id), @@ -1327,8 +1330,13 @@ stage VARCHAR(64), stage_id BIGINT, + component VARCHAR(64), imagedb VARCHAR(64), rlabel VARCHAR(64), - no_magic TINYINT, - PRIMARY KEY(dep_id) + outdir VARCHAR(255), + need_magic TINYINT, + PRIMARY KEY(dep_id), + KEY(state), + KEY(stage), + KEY(stage_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1342,4 +1350,8 @@ comment VARCHAR(255), PRIMARY KEY(target_id), + KEY(stage), + KEY(dist_group), + KEY(filter), + KEY(state), CONSTRAINT UNIQUE (dist_group, filter, stage, clean) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1365,4 +1377,9 @@ KEY(state), KEY(label), + KEY(stage), + KEY(stage_id), + KEY(magic_ds_id), + KEY(fault), + KEY(data_group), FOREIGN KEY(target_id) REFERENCES distTarget(target_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1549,5 +1566,5 @@ note VARCHAR(255), -- note registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered - PRIMARY KEY(dqstats_id), + PRIMARY KEY(ss_id), KEY(state), KEY(label), @@ -1581,5 +1598,15 @@ KEY(fault), KEY(quality), - FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id), + FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE Label ( + label VARCHAR(64), + priority INT, + active TINYINT, + comment VARCHAR(80), + PRIMARY KEY(label), + KEY(priority), + KEY(active) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: branches/pap/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/pap/ippTools/share/pxadmin_drop_tables.sql (revision 27708) +++ branches/pap/ippTools/share/pxadmin_drop_tables.sql (revision 28003) @@ -81,4 +81,8 @@ DROP TABLE IF EXISTS publishDone; DROP TABLE IF EXISTS pstampDependent; +DROP TABLE IF EXISTS staticskyInput; +DROP TABLE IF EXISTS staticskyResult; +DROP TABLE IF EXISTS staticskyRun; +DROP TABLE IF EXISTS Label; SET FOREIGN_KEY_CHECKS=1 Index: branches/pap/ippTools/share/stacktool_tosum.sql =================================================================== --- branches/pap/ippTools/share/stacktool_tosum.sql (revision 27708) +++ branches/pap/ippTools/share/stacktool_tosum.sql (revision 28003) @@ -6,5 +6,6 @@ stackRun.reduction, stackRun.label, - stackRun.state + stackRun.state, + stackSumSkyfile.path_base FROM stackRun JOIN stackInputSkyfile USING(stack_id) @@ -16,3 +17,3 @@ -- WHERE hook %s GROUP BY stack_id -HAVING COUNT(warpRun.state = 'full') = COUNT(stackInputSkyfile.warp_id) +HAVING SUM(IF(warpRun.state = 'full', 1, 0)) = COUNT(stackInputSkyfile.warp_id) Index: branches/pap/ippTools/share/warptool_listrun.sql =================================================================== --- branches/pap/ippTools/share/warptool_listrun.sql (revision 28003) +++ branches/pap/ippTools/share/warptool_listrun.sql (revision 28003) @@ -0,0 +1,21 @@ +SELECT + warpRun.*, + camRun.cam_id, + chipRun.chip_id, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM warpRun +JOIN fakeRun + ON warpRun.fake_id = fakeRun.fake_id +JOIN camRun + ON camRun.cam_id = fakeRun.cam_id +JOIN chipRun + ON camRun.chip_id = chipRun.chip_id +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id Index: branches/pap/ippTools/share/warptool_pendingcleanupskyfile.sql =================================================================== --- branches/pap/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 27708) +++ branches/pap/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 28003) @@ -10,5 +10,6 @@ USING(warp_id) WHERE - ((warpRun.state = 'goto_cleaned' AND warpSkyfile.data_state = 'full') + ((warpRun.state = 'goto_cleaned' AND (warpSkyfile.data_state = 'full' + OR warpSkyfile.data_state = 'update')) OR (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state != 'scrubbed') Index: branches/pap/ippTools/share/warptool_revertwarped_updated.sql =================================================================== --- branches/pap/ippTools/share/warptool_revertwarped_updated.sql (revision 28003) +++ branches/pap/ippTools/share/warptool_revertwarped_updated.sql (revision 28003) @@ -0,0 +1,10 @@ +UPDATE warpSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +SET warpSkyfile.fault = 0 +WHERE warpRun.state = 'update' + AND warpSkyfile.data_state = 'update' + AND warpSkyfile.fault != 0 Index: branches/pap/ippTools/share/warptool_scmap.sql =================================================================== --- branches/pap/ippTools/share/warptool_scmap.sql (revision 27708) +++ branches/pap/ippTools/share/warptool_scmap.sql (revision 28003) @@ -3,5 +3,10 @@ chipProcessedImfile.uri, chipProcessedImfile.path_base as chip_path_base, - camProcessedExp.path_base as cam_path_base + camProcessedExp.path_base as cam_path_base, + chipProcessedImfile.chip_id, + chipRun.state, + chipProcessedImfile.data_state, + chipProcessedImfile.magicked, + rawImfile.magicked AS raw_magicked FROM warpRun JOIN warpSkyCellMap @@ -18,7 +23,7 @@ ON chipRun.chip_id = chipProcessedImfile.chip_id AND warpSkyCellMap.class_id = chipProcessedImfile.class_id +JOIN rawImfile + ON chipRun.exp_id = rawImfile.exp_id + AND chipProcessedImfile.class_id = rawImfile.class_id WHERE --- warpRun.state = 'new' - fakeRun.state = 'full' - AND camRun.state = 'full' - AND chipRun.state = 'full' + 1 Index: branches/pap/ippTools/share/warptool_setskyfiletoupdate.sql =================================================================== --- branches/pap/ippTools/share/warptool_setskyfiletoupdate.sql (revision 28003) +++ branches/pap/ippTools/share/warptool_setskyfiletoupdate.sql (revision 28003) @@ -0,0 +1,8 @@ +UPDATE warpRun + JOIN warpSkyfile USING(warp_id) +SET warpRun.state = 'update', + warpSkyfile.data_state = 'update' + -- set hook %s +WHERE warp_id = %lld + AND (warpRun.state = 'cleaned' OR warpRun.state = 'update') + AND (warpSkyfile.data_state = 'cleaned') Index: branches/pap/ippTools/share/warptool_towarped.sql =================================================================== --- branches/pap/ippTools/share/warptool_towarped.sql (revision 27708) +++ branches/pap/ippTools/share/warptool_towarped.sql (revision 28003) @@ -1,2 +1,17 @@ +SELECT + warp_id, + warp_skyfile_id, + skycell_id, + tess_id, + fake_id, + state, + reduction, + cam_id, + camera, + exp_tag, + workdir, + magicked, + path_base +FROM ( SELECT DISTINCT warpSkyCellMap.warp_id, @@ -11,5 +26,6 @@ rawExp.exp_tag, warpRun.workdir, - chipRun.magicked + chipRun.magicked, + CAST(NULL AS CHAR(255)) AS path_base FROM warpRun JOIN warpSkyCellMap @@ -45,2 +61,41 @@ AND warpSkyCellMap.fault = 0 -- where hook 1 %s + -- limit hook 1 %s +UNION + SELECT + warpSkyCellMap.warp_id, + warpImfile.warp_skyfile_id, + warpSkyCellMap.skycell_id, + warpSkyCellMap.tess_id, + warpRun.fake_id, + warpRun.state, + warpRun.reduction, + camRun.cam_id, + rawExp.camera, + rawExp.exp_tag, + warpRun.workdir, + MIN(chipProcessedImfile.magicked) AS magicked, + warpSkyfile.path_base + FROM warpRun + JOIN warpImfile USING(warp_id) + JOIN warpSkyCellMap USING(warp_id, skycell_id) + JOIN warpSkyfile USING(warp_id, skycell_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN chipProcessedImfile USING(chip_id, class_id) + WHERE warpRun.state = 'update' + AND warpSkyfile.data_state = 'update' + AND warpSkyfile.fault = 0 + AND camRun.state = 'full' + AND chipProcessedImfile.data_state = 'full' + -- if warpSkyfile was magicked previously require inputs to be magicked + -- this blocks processing until all the chip inputs have been destreaked + AND (warpSkyfile.magicked = 0 OR chipProcessedImfile.magicked >= 0) + -- where hook 2 %s + GROUP BY warp_id, skycell_id + HAVING COUNT(warpSkyCellMap.class_id) = COUNT(chipProcessedImfile.class_id) + -- limit hook 2 %s +) as towarped + Index: branches/pap/ippTools/share/warptool_warped.sql =================================================================== --- branches/pap/ippTools/share/warptool_warped.sql (revision 27708) +++ branches/pap/ippTools/share/warptool_warped.sql (revision 28003) @@ -7,5 +7,10 @@ rawExp.exp_id, rawExp.exp_name, - rawExp.camera + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time FROM warpRun JOIN warpSkyfile @@ -21,6 +26,2 @@ JOIN rawExp ON chipRun.exp_id = rawExp.exp_id -WHERE --- bogus test; just here so there there is a 'WHERE' stmt to append conditionals too --- XXX EAM : not needed: fix warptool.c - warpRun.warp_id is NOT NULL Index: branches/pap/ippTools/src/addtool.c =================================================================== --- branches/pap/ippTools/src/addtool.c (revision 27708) +++ branches/pap/ippTools/src/addtool.c (revision 28003) @@ -280,4 +280,5 @@ PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-dvodb", "addRun.dvodb", "=="); if (!psListLength(where->list)) { Index: branches/pap/ippTools/src/addtoolConfig.c =================================================================== --- branches/pap/ippTools/src/addtoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/addtoolConfig.c (revision 28003) @@ -76,7 +76,9 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by addRun state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by addRun reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dvodb", 0, "search by dvodb", NULL); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dvodb", 0, "set dvodb", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); Index: branches/pap/ippTools/src/camtool.c =================================================================== --- branches/pap/ippTools/src/camtool.c (revision 27708) +++ branches/pap/ippTools/src/camtool.c (revision 28003) @@ -681,4 +681,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); // generate restrictions @@ -741,6 +742,10 @@ } - // order by cam_id so that the postage stamp parser can easliy find the 'latest' astrometry - psStringAppend(&query, " ORDER BY exp_id, cam_id"); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, " ORDER BY exp_id, cam_id DESC"); + } // treat limit == 0 as "no limit" Index: branches/pap/ippTools/src/camtoolConfig.c =================================================================== --- branches/pap/ippTools/src/camtoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/camtoolConfig.c (revision 28003) @@ -184,4 +184,6 @@ pxmagicAddArguments(processedexpArgs); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order the results for the postage stamp server", false); + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); Index: branches/pap/ippTools/src/chiptool.c =================================================================== --- branches/pap/ippTools/src/chiptool.c (revision 27708) +++ branches/pap/ippTools/src/chiptool.c (revision 28003) @@ -60,4 +60,5 @@ static bool runstateMode(pxConfig *config); static bool setimfiletoupdateMode(pxConfig *config); +static bool listrunMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -103,4 +104,5 @@ MODECASE(CHIPTOOL_MODE_RUNSTATE, runstateMode); MODECASE(CHIPTOOL_MODE_SETIMFILETOUPDATE, setimfiletoupdateMode); + MODECASE(CHIPTOOL_MODE_LISTRUN, listrunMode); default: psAbort("invalid option (this should not happen)"); @@ -420,8 +422,8 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); // we only allow a single label to match (do not use pxAddLabelSearchArgs here) - PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE"); PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); - PXOPT_COPY_STR(config->args, where, "-dist_group", "chipRun.dist_group", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "chipRun.dist_group", "LIKE"); if (!psListLength(where->list)) { @@ -469,4 +471,6 @@ } psFree(where); + + psStringAppend(&query, "\nORDER BY priority DESC, chip_id"); // treat limit == 0 as "no limit" @@ -699,4 +703,8 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(allfiles, config->args, "-allfiles", false); + if (allfiles) { + faulted = false; + } psMetadata *where = psMetadataAlloc(); @@ -740,5 +748,5 @@ // list only faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); - } else { + } else if (!allfiles) { // don't list faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); @@ -807,8 +815,15 @@ return false; } + psString query_update = pxDataGet("chiptool_revertupdatedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_update, " AND %s", whereClause); psFree(whereClause); } @@ -822,4 +837,10 @@ } psFree(query); + if (!p_psDBRunQuery(config->dbh, query_update)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_update); + return false; + } + psFree(query_update); return true; @@ -1414,6 +1435,6 @@ } - char *set_magicked_imfile = ""; - char *set_magicked_run = ""; + psString set_magicked_imfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); if (!strcmp(data_state, "full")) { // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will @@ -1421,10 +1442,14 @@ // otherwise copy magicked from the rawImfile // Same thing for chipRun/rawExp - set_magicked_imfile = "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0 AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)"; - set_magicked_run = "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), chipRun.magicked, rawExp.magicked)"; + psStringAppend(&set_magicked_imfile, "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0" + " AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)"); + psStringAppend(&set_magicked_run, "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), " + " chipRun.magicked, rawExp.magicked)"); + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { // if magicked is non-zero set it to -1 - set_magicked_imfile = "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)"; - set_magicked_run = "\n, chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"; + // Once one imfile has been cleaned, the chipRun is no longer 'magicked' + psStringAppend(&set_magicked_imfile, "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)," + " chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"); } @@ -1440,4 +1465,6 @@ } psFree(query); + psFree(set_magicked_imfile); + psFree(set_magicked_run); if (psDBAffectedRows(config->dbh) < 1) { psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); @@ -1757,2 +1784,90 @@ return true; } + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "chipRun.dist_group", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipRun"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + psString query = pxDataGet("chiptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); + psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || where2) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, chip_id DESC"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: branches/pap/ippTools/src/chiptool.h =================================================================== --- branches/pap/ippTools/src/chiptool.h (revision 27708) +++ branches/pap/ippTools/src/chiptool.h (revision 28003) @@ -50,5 +50,6 @@ CHIPTOOL_MODE_EXPORTRUN, CHIPTOOL_MODE_IMPORTRUN, - CHIPTOOL_MODE_RUNSTATE + CHIPTOOL_MODE_RUNSTATE, + CHIPTOOL_MODE_LISTRUN } chiptoolMode; Index: branches/pap/ippTools/src/chiptoolConfig.c =================================================================== --- branches/pap/ippTools/src/chiptoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/chiptoolConfig.c (revision 28003) @@ -196,4 +196,5 @@ psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-allfiles", 0, "only all matching imfiles regardless of fault status", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -216,4 +217,22 @@ psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + + // -processedExp + psMetadata *listrunArgs = psMetadataAlloc(); + pxchipSetSearchArgs(listrunArgs); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by chipRun state", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by chipRun dist_group (LIKE comparison)", NULL); + pxmagicAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + pxspaceAddArguments(listrunArgs); + // -block psMetadata *blockArgs = psMetadataAlloc(); @@ -328,4 +347,5 @@ PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); PXOPT_ADD_MODE("-runstate", "list the states of chip run", CHIPTOOL_MODE_RUNSTATE, runstateArgs); + PXOPT_ADD_MODE("-listrun", "list chipRuns", CHIPTOOL_MODE_LISTRUN, listrunArgs); PXOPT_ADD_MODE("-advanceexp", "advance completed exposures", CHIPTOOL_MODE_ADVANCEEXP, advanceexpArgs); PXOPT_ADD_MODE("-block", "set a label block", CHIPTOOL_MODE_BLOCK, blockArgs); Index: branches/pap/ippTools/src/difftool.c =================================================================== --- branches/pap/ippTools/src/difftool.c (revision 27708) +++ branches/pap/ippTools/src/difftool.c (revision 28003) @@ -57,4 +57,6 @@ static bool toscrubbedskyfileMode(pxConfig *config); static bool tofullskyfileMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); @@ -101,4 +103,6 @@ MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); + MODECASE(DIFFTOOL_MODE_LISTRUN, listrunMode); + MODECASE(DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); default: @@ -194,4 +198,5 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group","data_group", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); if (!psListLength(where->list)) { @@ -702,4 +707,5 @@ pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); if (!template) { @@ -745,4 +751,12 @@ psFree(where); + if (pstamp_order) { + if (template) { + psStringAppend(&query, " ORDER BY rawTemplate.exp_id, diff_id DESC"); + } else { + psStringAppend(&query, " ORDER BY rawInput.exp_id, diff_id DESC"); + } + } + // treat limit == 0 as "no limit" if (limit) { @@ -810,48 +824,13 @@ } - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Update state to 'new' - { - psString query = pxDataGet("difftool_revertdiffskyfile_update.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); - } - - // Delete product { psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } + return false; + } + psString query_updated = pxDataGet("difftool_revertdiffskyfile_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -860,4 +839,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); psFree(whereClause); } @@ -866,20 +846,21 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } return false; } psFree(query); - psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); + + psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); } psFree(where); - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } return true; @@ -971,16 +952,16 @@ return false; } - psS16 diff_mode = 0; + psS16 diff_mode = IPP_DIFF_MODE_UNDEFINED; if ((input_warp_id != PS_MAX_S64) && (template_warp_id != PS_MAX_S64)) { - diff_mode = 1; + diff_mode = IPP_DIFF_MODE_WARP_WARP; } else if ((input_warp_id != PS_MAX_S64) && (template_stack_id != PS_MAX_S64)) { - diff_mode = 2; + diff_mode = IPP_DIFF_MODE_WARP_STACK; } else if ((input_stack_id != PS_MAX_S64) && (template_warp_id != PS_MAX_S64)) { - diff_mode = 3; + diff_mode = IPP_DIFF_MODE_STACK_WARP; } else if ((input_stack_id != PS_MAX_S64) && (template_stack_id != PS_MAX_S64)) { - diff_mode = 4; + diff_mode = IPP_DIFF_MODE_STACK_STACK; } @@ -1366,5 +1347,4 @@ } - psString filter = psMetadataLookupStr(&mdok, row, "filter"); if (!mdok) { @@ -1378,4 +1358,20 @@ return false; } + + psString warp_data_group = psMetadataLookupStr(&mdok, row, "data_group"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "warp data_group not found"); + psFree(warp2Query); + psFree(stackQuery); + psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (warp_data_group) { + data_group = warp_data_group; + } + if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warp2Query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1416,5 +1412,5 @@ true, // exposure 0, // magicked - 2, // diff_mode for warp-stack difference + IPP_DIFF_MODE_WARP_STACK, note ); @@ -1739,4 +1735,9 @@ const char *template = psMetadataLookupStr(NULL, row, "template_warp_id"); const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); + psString input_data_group = psMetadataLookupStr(NULL, row, "input_data_group"); + if (input_data_group) { + data_group = input_data_group; + } + if (!input_id || !template || !tess_id) { psError(PXTOOLS_ERR_PROG, false, "Identifiers not found"); @@ -1749,5 +1750,5 @@ return false; } - + diffRunRow *run = diffRunRowAlloc(0, "reg", @@ -1763,5 +1764,5 @@ true, // exposure false, // magicked - 1, // diff_mode for warp-warp diff + IPP_DIFF_MODE_WARP_WARP, note); // Run to insert if (!diffRunInsertObject(config->dbh, run)) { @@ -2045,5 +2046,5 @@ false, // exposure 0, // magicked - 4, // diff_mode + IPP_DIFF_MODE_STACK_STACK, // diff_mode note ); @@ -2338,4 +2339,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) { @@ -2606,2 +2608,148 @@ return true; } + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "diffRun.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + if (!template) { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "LIKE"); + } else { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "LIKE"); + } + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "diffRun"); + pxspaceAddWhere(config, &where2, template ? "rawTemplate" : "rawInput"); + psString query = pxDataGet("difftool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (where2) { + psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + if (template) { + psStringAppend(&query, " ORDER BY rawTemplate.exp_id, diff_id DESC"); + } else { + psStringAppend(&query, " ORDER BY rawInput.exp_id, diff_id DESC"); + } + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("difftool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , diffRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} Index: branches/pap/ippTools/src/difftool.h =================================================================== --- branches/pap/ippTools/src/difftool.h (revision 27708) +++ branches/pap/ippTools/src/difftool.h (revision 28003) @@ -47,6 +47,7 @@ DIFFTOOL_MODE_TOPURGEDSKYFILE, DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, - DIFFTOOL_MODE_TOFULLSKYFILE - + DIFFTOOL_MODE_TOFULLSKYFILE, + DIFFTOOL_MODE_LISTRUN, + DIFFTOOL_MODE_SETSKYFILETOUPDATE, } difftoolMode; Index: branches/pap/ippTools/src/difftoolConfig.c =================================================================== --- branches/pap/ippTools/src/difftoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/difftoolConfig.c (revision 28003) @@ -65,4 +65,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define by label instead of diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "define by data_group instead of diff ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); @@ -147,11 +148,37 @@ psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL); psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); pxmagicAddArguments(diffskyfileArgs); pxspaceAddArguments(diffskyfileArgs); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", NULL); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-template", 0, "apply exposure args to template of bothways diff", false); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL); + pxmagicAddArguments(listrunArgs); + pxspaceAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -revertdiffskyfile @@ -299,5 +326,6 @@ psMetadata *updatediffskyfileArgs = psMetadataAlloc(); psMetadataAddS64(updatediffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); - psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); // -tocleanedskyfile @@ -321,4 +349,12 @@ psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); + + // -setskyfiletoupdate + psMetadata *setskyfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setskyfiletoupdateArgs, PS_LIST_TAIL, "-diff_id", 0, "search by difftool ID (required)", 0); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "new value for diffRun.label", NULL); + + // -exportrun @@ -348,4 +384,5 @@ PXOPT_ADD_MODE("-advance", "", DIFFTOOL_MODE_ADVANCE, advanceArgs); PXOPT_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); + PXOPT_ADD_MODE("-listrun", "", DIFFTOOL_MODE_LISTRUN, listrunArgs); PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); @@ -357,4 +394,6 @@ PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", DIFFTOOL_MODE_DONECLEANUP, donecleanupArgs); PXOPT_ADD_MODE("-updatediffskyfile", "update fault code for a diffskyfile", DIFFTOOL_MODE_UPDATEDIFFSKYFILE, updatediffskyfileArgs); + PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", DIFFTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", DIFFTOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/pap/ippTools/src/magicdstool.c =================================================================== --- branches/pap/ippTools/src/magicdstool.c (revision 27708) +++ branches/pap/ippTools/src/magicdstool.c (revision 28003) @@ -572,5 +572,4 @@ // we can transition by query as well - psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); @@ -1234,6 +1233,14 @@ psFree(whereClause); } else { - psError(PS_ERR_UNKNOWN, true, "search arugments are required"); - return false; + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; + } + + if (!strcmp(state, "goto_cleaned")) { + // Don't set state back to goto_cleaned if it is already cleaned + psStringAppend(&query, " AND (magicDSRun.state != 'cleaned')"); + + // don't clean up magicDSRun's where stage is camera + psStringAppend(&query, " AND (magicDSRun.stage != 'camera')"); } Index: branches/pap/ippTools/src/magictool.c =================================================================== --- branches/pap/ippTools/src/magictool.c (revision 27708) +++ branches/pap/ippTools/src/magictool.c (revision 28003) @@ -48,4 +48,5 @@ static bool maskMode(pxConfig *config); static bool censorrunMode(pxConfig *config); +static bool exposureMode(pxConfig *config); static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -87,4 +88,5 @@ MODECASE(MAGICTOOL_MODE_MASK, maskMode); MODECASE(MAGICTOOL_MODE_CENSORRUN, censorrunMode); + MODECASE(MAGICTOOL_MODE_EXPOSURE, exposureMode); default: psAbort("invalid option (this should not happen)"); @@ -1514,2 +1516,66 @@ return true; } + + +static bool exposureMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_BOOL(config->args, where, "-inverse", "magicRun.inverse", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_exposure.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psStringCopy(""); // WHERE restrictions + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\nWHERE %s", clause); + psFree(clause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "magicMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} Index: branches/pap/ippTools/src/magictool.h =================================================================== --- branches/pap/ippTools/src/magictool.h (revision 27708) +++ branches/pap/ippTools/src/magictool.h (revision 28003) @@ -42,4 +42,5 @@ MAGICTOOL_MODE_MASK, MAGICTOOL_MODE_CENSORRUN, + MAGICTOOL_MODE_EXPOSURE, } MAGICtoolMode; Index: branches/pap/ippTools/src/magictoolConfig.c =================================================================== --- branches/pap/ippTools/src/magictoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/magictoolConfig.c (revision 28003) @@ -168,4 +168,10 @@ psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); psMetadataAddStr(censorrunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + + // -exposure + psMetadata *exposureArgs = psMetadataAlloc(); + psMetadataAddS64(exposureArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); + psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-inverse", 0, "select the inverse subtraction?", false); + psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -191,4 +197,5 @@ PXOPT_ADD_MODE("-mask", "", MAGICTOOL_MODE_MASK, maskArgs); PXOPT_ADD_MODE("-censorrun", "", MAGICTOOL_MODE_CENSORRUN, censorrunArgs); + PXOPT_ADD_MODE("-exposure", "", MAGICTOOL_MODE_EXPOSURE, exposureArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/pap/ippTools/src/pstamptool.c =================================================================== --- branches/pap/ippTools/src/pstamptool.c (revision 27708) +++ branches/pap/ippTools/src/pstamptool.c (revision 28003) @@ -40,4 +40,5 @@ static bool updatereqMode(pxConfig *config); static bool revertreqMode(pxConfig *config); +static bool pendingcleanupMode(pxConfig *config); static bool addjobMode(pxConfig *config); static bool listjobMode(pxConfig *config); @@ -51,4 +52,5 @@ static bool pendingdependentMode(pxConfig *config); static bool updatedependentMode(pxConfig *config); +static bool revertdependentMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -80,4 +82,5 @@ MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updatereqMode); MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertreqMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGCLEANUP, pendingcleanupMode); MODECASE(PSTAMPTOOL_MODE_ADDJOB, addjobMode); MODECASE(PSTAMPTOOL_MODE_LISTJOB, listjobMode); @@ -91,4 +94,5 @@ MODECASE(PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentMode); MODECASE(PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentMode); + MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode); default: psAbort("invalid option (this should not happen)"); @@ -263,4 +267,5 @@ NULL, // outProduct uri, + NULL, // outdir 0 // fault )) { @@ -299,4 +304,6 @@ } psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); // treat limit == 0 as "no limit" @@ -337,5 +344,5 @@ } -static bool listreqMode(pxConfig *config) +static bool pendingcleanupMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -343,21 +350,23 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); - PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - if (!psListLength(where->list)) { - psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); - return false; - } - - psString query = psStringCopy("SELECT * from pstampRequest"); - - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - psFree(where); + psString query = pxDataGet("pstamptool_pendingcleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampRequest"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); // treat limit == 0 as "no limit" @@ -381,10 +390,7 @@ } if (!psArrayLength(output)) { - psTrace("pstamptool", PS_LOG_INFO, "request not found"); - // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for - psError(PXTOOLS_ERR_CONFIG, true, "request not found"); - psFree(output); - // we return false so that the caller can determine that a request does not exist - return false; + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; } @@ -401,21 +407,26 @@ } -static bool completedreqMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); +static bool listreqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = pxDataGet("pstamptool_completedreq.sql"); - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } + if (!psListLength(where->list)) { + psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); + return false; + } + + psString query = psStringCopy("SELECT * from pstampRequest"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); psFree(where); @@ -432,4 +443,5 @@ return false; } + psFree(query); psArray *output = p_psDBFetchResult(config->dbh); @@ -438,9 +450,11 @@ return false; } - if (!psArrayLength(output)) { - psTrace("pstamptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; + psTrace("pstamptool", PS_LOG_INFO, "request not found"); + // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for + psError(PXTOOLS_ERR_CONFIG, true, "request not found"); + psFree(output); + // we return false so that the caller can determine that a request does not exist + return false; } @@ -457,23 +471,103 @@ } +static bool completedreqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_completedreq.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + static bool updatereqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); - PXOPT_LOOKUP_STR(outProduct, config->args, "-outProduct", false, false); - PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); - PXOPT_LOOKUP_STR(reqType, config->args, "-reqType", false, false); - - psString query = NULL; - psStringAppend(&query, "UPDATE pstampRequest SET"); - + // PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-set_outProduct", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-set_uri", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-set_name", false, false); + PXOPT_LOOKUP_STR(reqType, config->args, "-set_reqType", false, false); + PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id_max", "req_id", "<="); + PXOPT_COPY_S32(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE pstampRequest SET"); + + psString stateCheck = NULL; char c = ' '; if (state) { psStringAppend(&query, "%c state = '%s'", c, state); c = ','; + if (!strcmp(state, "goto_cleaned")) { + psStringAppend(&stateCheck, " AND state != 'cleaned'"); + } + } + if (label) { + psStringAppend(&query, "%c label = '%s'", c, label); + c = ','; } if (outProduct) { @@ -481,6 +575,17 @@ c = ','; } - if (fault) { - psStringAppend(&query, "%c fault = %s", c, fault); + if (outdir) { + psStringAppend(&query, "%c outdir = '%s'", c, outdir); + c = ','; + } + if (clearfault) { + if (fault) { + psError(PXTOOLS_ERR_CONFIG, true, "only one of -fault and -clearfault is allowed"); + return false; + } + psStringAppend(&query, "%c fault = 0", c); + c = ','; + } else if (fault) { + psStringAppend(&query, "%c fault = %d", c, fault); c = ','; } @@ -497,10 +602,17 @@ c = ','; } - if (!state && !outProduct && !fault && !uri && !name && !reqType) { - psError(PS_ERR_UNKNOWN, true, "at least one of state, outProduct, fault, uri, name, and reqType must be specified"); - return false; - } - - psStringAppend(&query, " WHERE req_id = %" PRId64, req_id); + if (c != ',') { + psError(PS_ERR_UNKNOWN, true, "at least one set option is required"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (stateCheck) { + psStringAppend(&query, "%s", stateCheck); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -510,11 +622,6 @@ } - psU64 affected = psDBAffectedRows(config->dbh); - // note zero is not an error - if (affected > 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" - PRIu64 " rows were modified", affected); - return false; - } + // psU64 affected = psDBAffectedRows(config->dbh); + // psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampRequests", affected); return true; @@ -528,5 +635,5 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "=="); pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); @@ -703,5 +810,5 @@ PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -722,4 +829,6 @@ psFree(where); + psStringAppend(&query, " ORDER BY priority DESC, req_id"); + // treat limit == 0 as "no limit" if (limit) { @@ -763,32 +872,51 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); - - psString faultStr = NULL; - if (!fault) { - faultStr = psStringCopy(""); - } else { - psStringAppend(&faultStr, ", fault = '%s'", fault); - } - - char *query ="UPDATE pstampJob" - " SET state = '%s' %s" - " WHERE job_id = %" PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, faultStr, job_id)) { + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false); + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); + + if (!job_id && !dep_id) { + psError(PS_ERR_UNKNOWN, true, "at least -job_id or -dep_id is required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_S32(fault, config->args, "-set_fault", false, false); + + if (!state && !fault) { + psError(PS_ERR_UNKNOWN, true, "at least one of -set_state and -set_fault is required"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + + psString query = pxDataGet("pstamptool_updatejob.sql"); + + char c = ' '; + if (state) { + psStringAppend(&query, "\n %c pstampJob.state = '%s'", c, state); + c = ','; + } + if (fault) { + psStringAppend(&query, "\n %c pstampJob.fault = %d", c, fault); + c = ','; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); return false; } - psFree(faultStr); + psFree(query); psU64 affected = psDBAffectedRows(config->dbh); - if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" - PRIu64 " rows were modified", affected); - return false; - } + psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampJobs", affected); return true; @@ -952,7 +1080,9 @@ PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); PXOPT_LOOKUP_STR(imagedb, config->args, "-imagedb", true, false); - PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", false, false); - PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", true, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", true, false); + PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false); PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); @@ -961,4 +1091,5 @@ PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "=="); PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); // start a transaction eraly so it will contain any row level locks @@ -968,15 +1099,15 @@ } - psString query = pxDataGet("pstamptool_pendingdependent.sql"); + psString query = pxDataGet("pstamptool_getdependent.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } - psString whereClause = psDBGenerateWhereSQL(where, "WHERE"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); if (!no_create) { // This will lock the row until the transaction is committed - psStringAppend(&query, " %s FOR UPDATE", whereClause); + psStringAppend(&query, " AND %s FOR UPDATE", whereClause); } else { - psStringAppend(&query, " %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); } psFree(whereClause); @@ -1027,7 +1158,10 @@ stage, stage_id, + component, imagedb, rlabel, - no_magic + need_magic, + outdir, + 0 // fault )) { if (!psDBRollback(config->dbh)) { @@ -1038,4 +1172,14 @@ } + // if we try and get this after commit zero is returned + psS64 dep_id = psDBLastInsertID(config->dbh); + if (!dep_id) { + psError(PS_ERR_UNKNOWN, false, "psDBLastInsertID returned NULL"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psDBCommit(config->dbh)) { // rollback @@ -1047,5 +1191,4 @@ } - psS64 dep_id = psDBLastInsertID(config->dbh); printf("%" PRId64 "\n", dep_id); @@ -1058,4 +1201,10 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + PXOPT_COPY_STR(config->args, where, "-rlabel", "rlabel", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -1069,9 +1218,11 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampDependent"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); // treat limit == 0 as "no limit" @@ -1116,11 +1267,24 @@ PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); - - char *query ="UPDATE pstampDependent" - " SET state = '%s'" - " WHERE dep_id = %" PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, dep_id)) { + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + + if (!fault && !state) { + psError(PS_ERR_UNKNOWN, true, "at least one of -set_state or fault is required"); + return false; + } + psString query = psStringCopy("UPDATE pstampDependent SET"); + bool needComma = false; + if (state) { + psStringAppend(&query, " state = '%s'", state); + needComma = true; + } + if (fault) { + psStringAppend(&query, "%s fault = %d", needComma ? ", " : "", fault); + needComma = true; + } + psStringAppend(&query, " WHERE dep_id = %" PRId64, dep_id); + + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1137,2 +1301,35 @@ return true; } +static bool revertdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fault", "pstampDependent.fault", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = pxDataGet("pstamptool_revertdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} Index: branches/pap/ippTools/src/pstamptool.h =================================================================== --- branches/pap/ippTools/src/pstamptool.h (revision 27708) +++ branches/pap/ippTools/src/pstamptool.h (revision 28003) @@ -34,4 +34,5 @@ PSTAMPTOOL_MODE_UPDATEREQ, PSTAMPTOOL_MODE_REVERTREQ, + PSTAMPTOOL_MODE_PENDINGCLEANUP, PSTAMPTOOL_MODE_ADDJOB, PSTAMPTOOL_MODE_LISTJOB, @@ -46,4 +47,5 @@ PSTAMPTOOL_MODE_PENDINGDEPENDENT, PSTAMPTOOL_MODE_UPDATEDEPENDENT, + PSTAMPTOOL_MODE_REVERTDEPENDENT, } pstamptoolMode; Index: branches/pap/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/pap/ippTools/src/pstamptoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/pstamptoolConfig.c (revision 28003) @@ -98,11 +98,18 @@ // -updatereq psMetadata *updatereqArgs = psMetadataAlloc(); - psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to update", 0); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id_max", 0, "maximum req_id for which to update", 0); + psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_state", 0, "new state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_label", 0, "new label", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_outProduct", 0, "new outProduct", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_outdir", 0, "new outdir", NULL); + psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-set_fault", 0, "new fault code", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_uri", 0, "new uri", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_name", 0, "new name", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_reqType", 0, "new reqType", NULL); + psMetadataAddBool(updatereqArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); // -revertreq @@ -112,4 +119,12 @@ psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); + + // -pendingcleanup + psMetadata *pendingcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddStr(pendingcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); + psMetadataAddU64(pendingcleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -addjob @@ -144,6 +159,7 @@ psMetadata *updatejobArgs = psMetadataAlloc(); psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id for which to change state", 0); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0, "new state", NULL); + psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0, "new result", 0); // -revertjob @@ -159,24 +175,36 @@ // -getdependent psMetadata *getdependentArgs = psMetadataAlloc(); - psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run (required)", 0); - psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run (required)", NULL); - psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run (required)", NULL); - psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); - psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); - psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + // arguabely all of these could be -set_ arguments, but since this mode is both a query and + // a create if doesn't exist I like it this way + psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage id for dependent (required)", 0); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for depenent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent ", NULL); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-need_magic", 0, "define need_magic", false); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory for dependent (required)", NULL); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no matching dependent do not create one", false); // -updatedependent psMetadata *updatedependentArgs = psMetadataAlloc(); - psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent run (required)", 0); - psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); + psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent (required)", 0); + psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); + psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-set_fault", 0, "new value for fault", 0); + + // -revertdependent + psMetadata *revertdependentArgs = psMetadataAlloc(); + psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent", 0); + psMetadataAddStr(revertdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for dependent ", NULL); + psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault", 0, "define fault", 0); // -pendingdependent psMetadata *pendingdependentArgs = psMetadataAlloc(); - psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run", 0); - psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run", NULL); - psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run", NULL); - psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); - psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); - psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent", 0); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for dependent", NULL); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define dep_id for dependent", 0); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for pstampRequest", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent", NULL); psMetadataAddU64(pendingdependentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingdependentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -216,4 +244,5 @@ PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); PXOPT_ADD_MODE("-revertreq", "", PSTAMPTOOL_MODE_REVERTREQ, revertreqArgs); + PXOPT_ADD_MODE("-pendingcleanup", "", PSTAMPTOOL_MODE_PENDINGCLEANUP, pendingcleanupArgs); PXOPT_ADD_MODE("-addjob", "", PSTAMPTOOL_MODE_ADDJOB, addjobArgs); @@ -230,4 +259,5 @@ PXOPT_ADD_MODE("-updatedependent", "", PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentArgs); PXOPT_ADD_MODE("-pendingdependent","", PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentArgs); + PXOPT_ADD_MODE("-revertdependent","", PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentArgs); PXOPT_ADD_MODE("-addproject", "", PSTAMPTOOL_MODE_ADDPROJECT, addprojectArgs); Index: branches/pap/ippTools/src/pxchip.c =================================================================== --- branches/pap/ippTools/src/pxchip.c (revision 27708) +++ branches/pap/ippTools/src/pxchip.c (revision 28003) @@ -69,9 +69,9 @@ psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); - psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); return true; } @@ -121,5 +121,5 @@ PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "LIKE"); PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); Index: branches/pap/ippTools/src/pxmagic.c =================================================================== --- branches/pap/ippTools/src/pxmagic.c (revision 27708) +++ branches/pap/ippTools/src/pxmagic.c (revision 28003) @@ -85,6 +85,7 @@ } if (destreaked) { - psStringAppend(out, " AND %s.magicked > 0", table); + psStringAppend(out, " AND %s.magicked != 0", table); } + // Note -magicked is handled by the caller. XXX: Why? return true; } Index: branches/pap/ippTools/src/pxtools.h =================================================================== --- branches/pap/ippTools/src/pxtools.h (revision 27708) +++ branches/pap/ippTools/src/pxtools.h (revision 28003) @@ -286,4 +286,6 @@ PXOPT_COPY_PRIMITIVE(from, to, psU64, U64, oldname, newname, comment) +#define PXOPT_COPY_BOOL(from, to, oldname, newname, comment) \ + PXOPT_COPY_V(from, to, psBool, Bool, oldname, newname, comment) /*** these PXOPT_ADD_WHERE macros are used to construct the default sql elements ***/ Index: branches/pap/ippTools/src/regtool.c =================================================================== --- branches/pap/ippTools/src/regtool.c (revision 27708) +++ branches/pap/ippTools/src/regtool.c (revision 28003) @@ -350,4 +350,8 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(allfiles, config->args, "-allfiles", false); + if (allfiles) { + faulted = false; + } PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -389,5 +393,5 @@ // list only faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault != 0"); - } else { + } else if (!allfiles) { // don't list faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); Index: branches/pap/ippTools/src/regtoolConfig.c =================================================================== --- branches/pap/ippTools/src/regtoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/regtoolConfig.c (revision 28003) @@ -144,4 +144,5 @@ ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); + ADD_OPT(Bool, processedimfileArgs, "-allfiles", "return imfiles regardless of fault status", false); ADD_OPT(Bool, processedimfileArgs, "-all", "list without search arguments", false); ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); Index: branches/pap/ippTools/src/stacktool.c =================================================================== --- branches/pap/ippTools/src/stacktool.c (revision 27708) +++ branches/pap/ippTools/src/stacktool.c (revision 28003) @@ -155,4 +155,5 @@ PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-select_data_group", "warpRun.data_group", "=="); pxAddLabelSearchArgs (config, where, "-select_label", "warpRun.label", "LIKE"); // define using warp label Index: branches/pap/ippTools/src/stacktoolConfig.c =================================================================== --- branches/pap/ippTools/src/stacktoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/stacktoolConfig.c (revision 28003) @@ -59,4 +59,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", "object"); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by warpRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_data_group", 0 , "search by warpRun data_group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_comment", 0, "search for comment (LIKE)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); Index: branches/pap/ippTools/src/staticskytool.c =================================================================== --- branches/pap/ippTools/src/staticskytool.c (revision 27708) +++ branches/pap/ippTools/src/staticskytool.c (revision 28003) @@ -107,9 +107,9 @@ // optional - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); - PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); @@ -118,10 +118,10 @@ PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); - PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "stackRun.skycell_id", "=="); - pxAddLabelSearchArgs(config, where, "-select_label", "stackRun.label", "LIKE"); - pxAddLabelSearchArgs(config, where, "-select_filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "stackRun.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-select_filter", "stackRun.filter", "LIKE"); // these are used to build the HAVING restriction - PXOPT_COPY_S32(config->args, having, "-min_num", "num", ">="); + PXOPT_COPY_S32(config->args, having, "-min_num", "num", ">="); PXOPT_COPY_S32(config->args, having, "-min_filter", "num_filter", ">="); Index: branches/pap/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/pap/ippTools/src/staticskytoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/staticskytoolConfig.c (revision 28003) @@ -46,17 +46,18 @@ // -definebyquery + // XXX need a 'ra_min,max', 'dec_min,max' : to do this, we need a table of skycell boundaries psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -64,5 +65,5 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ss_id", 0, "search by stack ID", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by stack ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); @@ -75,5 +76,5 @@ // -inputs psMetadata *inputsArgs = psMetadataAlloc(); - psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -81,5 +82,5 @@ // -todo psMetadata *todoArgs = psMetadataAlloc(); - psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -88,5 +89,5 @@ // -addresult psMetadata *addresultArgs = psMetadataAlloc(); - psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ss_id", 0, "define stack ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-sky_id", 0, "define stack ID (required)", 0); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN); @@ -100,5 +101,5 @@ // -result psMetadata *resultArgs= psMetadataAlloc(); - psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); @@ -111,5 +112,5 @@ // -revert psMetadata *revertArgs= psMetadataAlloc(); - psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -118,10 +119,10 @@ // -updateresult psMetadata *updateresultArgs = psMetadataAlloc(); - psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ss_id", 0, "define staticksky ID (required)", 0); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-sky_id", 0, "define staticksky ID (required)", 0); psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); - psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ss_id", 0, "export this staticsky ID (required)", 0); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-sky_id", 0, "export this staticsky ID (required)", 0); psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/pap/ippTools/src/warptool.c =================================================================== --- branches/pap/ippTools/src/warptool.c (revision 27708) +++ branches/pap/ippTools/src/warptool.c (revision 28003) @@ -59,4 +59,6 @@ static bool importrunMode(pxConfig *config); static bool runstateMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); @@ -110,4 +112,6 @@ MODECASE(WARPTOOL_MODE_IMPORTRUN, importrunMode); MODECASE(WARPTOOL_MODE_RUNSTATE, runstateMode); + MODECASE(WARPTOOL_MODE_LISTRUN, listrunMode); + MODECASE(WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); default: @@ -244,4 +248,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); @@ -269,4 +275,5 @@ PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); // check mode @@ -306,4 +313,15 @@ if (!psArrayLength(output)) { psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } psFree(output); return true; @@ -916,12 +934,16 @@ // treat limit == 0 as "no limit" + psString limitString = psStringCopy("\n"); if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); + // We apply the limit to both sides of the UNION + // to avoid slow queries and to the query itself + // to satisfy the user's requested limit + psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - + } + +#define TWO_WHERES #ifdef TWO_WHERES - if (!p_psDBRunQueryF(config->dbh, query, whereStr, whereStr)) { + if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr, limitString)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -935,4 +957,5 @@ } #endif + psFree(limitString); psFree(whereStr); psFree(query); @@ -994,5 +1017,5 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); // default values @@ -1242,5 +1265,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); } else if (!all && !where2) { @@ -1248,9 +1271,13 @@ return false; } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } + } psFree(where); - - if (where2) { - psStringAppend(&query, " %s", where2); - } // treat limit == 0 as "no limit" @@ -1322,49 +1349,46 @@ } - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Delete product - int numDeleted; // Number deleted - { - psString query = pxDataGet("warptool_revertwarped_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psString query = pxDataGet("warptool_revertwarped_delete.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString query_updated = pxDataGet("warptool_revertwarped_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkyfiles", numDeleted); - psFree(where); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + // fix any faulted warpSkyfiles in data_state 'update' + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("warptool", PS_LOG_INFO, "Updated %d warpSkyfiles", numUpdated); return true; @@ -1665,13 +1689,17 @@ } - char *set_magicked_skyfile = ""; - char *set_magicked_run = ""; + psString set_magicked_skyfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); if (!strcmp(data_state, "full")) { - set_magicked_skyfile = "\n , warpSkyfile.magicked = IF((warpSkyfile.magicked < 0 AND chipRun.magicked = 0), warpSkyfile.magicked, chipRun.magicked)"; - set_magicked_run = "\n, warpRun.magicked = IF((warpRun.magicked < 0 AND chipRun.magicked = 0), warpRun.magicked, chipRun.magicked)"; - + // magicked is only an argument for for -tofullskyfile + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + if (magicked) { + psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked); + psStringAppend(&set_magicked_run, "\n, warpRun.magicked = %" PRId64, magicked); + } } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { - set_magicked_skyfile = "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1)"; - set_magicked_run = "\n, warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)"; + // if magicked is currently nonzero set it to -1 + // Set warpRun.magicked when the first skyfile is cleaned + psStringAppend(&set_magicked_skyfile, "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1), warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)"); } @@ -1686,4 +1714,5 @@ } psFree(query); + psFree(set_magicked_skyfile); query = pxDataGet("warptool_change_run_state.sql"); @@ -1696,4 +1725,5 @@ return false; } + psFree(set_magicked_run); if (!psDBCommit(config->dbh)) { @@ -2026,2 +2056,154 @@ return true; } + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpRun.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "warpRun.dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpRun")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, warp_id DESC"); + } + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("warptool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , warpRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} Index: branches/pap/ippTools/src/warptool.h =================================================================== --- branches/pap/ippTools/src/warptool.h (revision 27708) +++ branches/pap/ippTools/src/warptool.h (revision 28003) @@ -55,4 +55,6 @@ WARPTOOL_MODE_IMPORTRUN, WARPTOOL_MODE_RUNSTATE, + WARPTOOL_MODE_LISTRUN, + WARPTOOL_MODE_SETSKYFILETOUPDATE, } warptoolMode; Index: branches/pap/ippTools/src/warptoolConfig.c =================================================================== --- branches/pap/ippTools/src/warptoolConfig.c (revision 27708) +++ branches/pap/ippTools/src/warptoolConfig.c (revision 28003) @@ -89,6 +89,8 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by fakeRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on fakeRun label", NULL); @@ -106,4 +108,5 @@ psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); // -definerun @@ -217,5 +220,5 @@ psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); - psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-magicked", 0, "define if this skycell has been magicked", 0); + psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-set_magicked", 0, "define if this skycell has been magicked", 0); psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); @@ -236,4 +239,5 @@ psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL); // add magic related arguments pxmagicAddArguments(warpedArgs); @@ -244,4 +248,30 @@ psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); + psMetadataAddS16(listrunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL); + // add magic related arguments + pxmagicAddArguments(listrunArgs); + // add arguments for spatial search + pxspaceAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertwarped @@ -315,4 +345,5 @@ psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-set_magicked", 0, "new value for magicked", 0); // -updateskyfile @@ -345,4 +376,10 @@ psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -setskyfiletoupdate + psMetadata *setskyfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setskyfiletoupdateArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID (required)", 0); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "new value for warpRun.label", NULL); @@ -364,4 +401,5 @@ PXOPT_ADD_MODE("-advancerun", "", WARPTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-listrun", "", WARPTOOL_MODE_LISTRUN, listrunArgs); PXOPT_ADD_MODE("-runstate", "", WARPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); @@ -377,4 +415,5 @@ PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); + PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", WARPTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", WARPTOOL_MODE_IMPORTRUN, importrunArgs);