Index: branches/eam_branches/ipp-20130711/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/Makefile.am (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/Makefile.am (revision 36071) @@ -193,4 +193,5 @@ disttool_definebyquery_sky.sql \ disttool_definebyquery_sky_singlefilter.sql \ + disttool_definebyquery_skycal.sql \ disttool_definebyquery_stack.sql \ disttool_definebyquery_warp.sql \ @@ -207,4 +208,5 @@ disttool_pending_raw.sql \ disttool_pending_sky.sql \ + disttool_pending_skycal.sql \ disttool_pending_stack.sql \ disttool_pending_warp.sql \ @@ -398,4 +400,5 @@ stacktool_tosummary.sql \ stacktool_addsummary.sql \ + stacktool_summary.sql \ staticskytool_definebyquery_select.sql \ staticskytool_definebyquery_select_by_dg.sql \ @@ -408,4 +411,6 @@ staticskytool_export_input.sql \ staticskytool_export_result.sql \ + staticskytool_export_skycalrun.sql \ + staticskytool_export_skycalresult.sql \ staticskytool_inputs.sql \ staticskytool_todo.sql \ @@ -476,8 +481,10 @@ releasetool_definerelstack_with_skycal.sql \ releasetool_listrelstack.sql \ + releasetool_summary.sql \ releasetool_definerelgroup_select_lap.sql \ releasetool_definerelgroup_select_data_group.sql \ releasetool_definerelgroup_select_exp_data_group.sql \ releasetool_definerelgroup_select_exp_lap.sql \ + releasetool_stacksummary.sql \ releasetool_pendingrelgroup.sql Index: branches/eam_branches/ipp-20130711/ippTools/share/camtool_pendingexp.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/camtool_pendingexp.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/camtool_pendingexp.sql (revision 36071) @@ -23,5 +23,6 @@ WHERE chipRun.state = 'full' - AND ((camRun.state = 'new' AND camProcessedExp.cam_id IS NULL) OR camRun.state = 'update') + AND ((camRun.state = 'new' AND camProcessedExp.cam_id IS NULL) OR + (camRun.state = 'update' and camProcessedExp.fault = 0 and camProcessedExp.quality = 0)) AND camMask.label IS NULL AND (Label.active OR Label.active IS NULL) Index: branches/eam_branches/ipp-20130711/ippTools/share/disttool_definebyquery_sky.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/disttool_definebyquery_sky.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/disttool_definebyquery_sky.sql (revision 36071) @@ -3,5 +3,5 @@ staticskyRun.sky_id AS stage_id, CAST(0 AS SIGNED) AS magicked, - -- run tag in the form 'sky.$skycell_id.$stack_id' + -- run tag in the form 'sky.$skycell_id.$sky_id' CONCAT_WS('.', 'sky', stackRun.skycell_id, convert(staticskyRun.sky_id, CHAR)) as run_tag, staticskyRun.label, Index: branches/eam_branches/ipp-20130711/ippTools/share/disttool_definebyquery_skycal.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/disttool_definebyquery_skycal.sql (revision 36071) +++ branches/eam_branches/ipp-20130711/ippTools/share/disttool_definebyquery_skycal.sql (revision 36071) @@ -0,0 +1,27 @@ +SELECT + 'skycal' as stage, + skycalRun.skycal_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'skycal.$skycell_id.$skycal_id' + CONCAT_WS('.', 'skycal', stackRun.skycell_id, convert(skycalRun.skycal_id, CHAR)) as run_tag, + skycalRun.label, + skycalRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM skycalRun +JOIN skycalResult USING(skycal_id) +JOIN stackRun using(stack_id) +JOIN distTarget ON distTarget.stage = 'skycal' + AND skycalRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = skycal_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL + AND ((skycalRun.state = 'full') OR (distTarget.clean AND skycalRun.state = 'cleaned')) + AND stackRun.filter = distTarget.filter + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (skycalResult.fault = 0 AND skycalResult.quality = 0) Index: branches/eam_branches/ipp-20130711/ippTools/share/disttool_pending_skycal.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/disttool_pending_skycal.sql (revision 36071) +++ branches/eam_branches/ipp-20130711/ippTools/share/disttool_pending_skycal.sql (revision 36071) @@ -0,0 +1,52 @@ +SELECT DISTINCT + distRun.dist_id, + distRun.label, + distTarget.dist_group, + stage, + stage_id, + stackRun.skycell_id AS component, + exp_type, + clean, + rawExp.camera, + CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir, + skycalResult.path_base, + CAST(NULL AS CHAR(255)) as chip_path_base, + skycalRun.state, + skycalRun.state AS data_state, + skycalResult.quality, + 1 AS no_magic, + 0 AS magicked, + IFNULL(Label.priority, 10000) AS priority +FROM distRun +JOIN distTarget USING(target_id, stage, clean) +JOIN skycalRun + ON stage_id = skycal_id +JOIN skycalResult + USING(skycal_id) +JOIN stackRun USING(stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING (exp_id) +LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + AND stackRun.skycell_id = distComponent.component +LEFT JOIN Label ON distRun.label = Label.label +WHERE + distRun.state = 'new' + AND distRun.stage = 'skycal' + AND distComponent.dist_id IS NULL + AND (skycalRun.state = 'full' OR (distRun.clean AND skycalRun.state = 'cleaned')) + AND (skycalResult.fault = 0 AND skycalResult.quality = 0) + AND (Label.active OR Label.active IS NULL) Index: branches/eam_branches/ipp-20130711/ippTools/share/disttool_toadvance.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/disttool_toadvance.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/disttool_toadvance.sql (revision 36071) @@ -262,4 +262,25 @@ AND distComponent.fault = 0 UNION +-- skycal stage +-- NOTE this assumes that there is only one component per skycalRun +-- (one skycell) +SELECT + distRun.dist_id, + stage, + stage_id, + outroot, + label, + clean + FROM distRun + JOIN skycalResult on stage_id = skycal_id + LEFT JOIN distComponent + ON distRun.dist_id = distComponent.dist_id + WHERE + distRun.state = 'new' + AND distRun.fault = 0 + AND distRun.stage = 'skycal' + AND distComponent.component IS NOT NULL + AND distComponent.fault = 0 +UNION -- SSdiff stage SELECT Index: branches/eam_branches/ipp-20130711/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/pxadmin_create_tables.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/pxadmin_create_tables.sql (revision 36071) @@ -2172,4 +2172,6 @@ dataRelease INT, -- PSPS dataRelease priority INT, + dvodb VARCHAR(255), + ubercal_file VARCHAR(255), PRIMARY KEY(rel_id), UNIQUE KEY(surveyID, release_name), @@ -2186,9 +2188,11 @@ chip_id BIGINT, -- links to the runs that supplied the data cam_id BIGINT, -- for this release - group_id INT, -- id of relGroup that this exposure is contained in + group_id INT, -- id of relGroup that this exposure is contained in state VARCHAR(16), -- released, pending, archive, drop - flags INT UNSIGNED, -- flags for relphot, relastro, ?? + flags INT UNSIGNED, -- flags for relphot, relastro, ?? zpt_obs FLOAT, -- calibrated zero point for this release of zpt_stdev FLOAT, -- this exposure + mcal FLOAT, -- this exposure + ubercal_dist INT, -- ubercal distance (from dvo calibration) path_base VARCHAR(255), -- path_base of any supporting files for this -- release of this exposure. @@ -2273,4 +2277,33 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE dqstatsRun ( + dqstats_id BIGINT NOT NULL AUTO_INCREMENT, + state VARCHAR(64) DEFAULT NULL, + registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + label VARCHAR(64) DEFAULT NULL, + fault SMALLINT NOT NULL, + PRIMARY KEY (dqstats_id), + KEY label (label), + KEY fault (fault) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE dqstatsContent ( + dqstats_id BIGINT, + exp_id BIGINT, + chip_id BIGINT, + cam_id BIGINT, + warp_id BIGINT, + invalid TINYINT DEFAULT NULL, + KEY dqstats_id (dqstats_id), + KEY exp_id (exp_id), + KEY chip_id (chip_id), + KEY cam_id (cam_id), + KEY warp_id (warp_id), + FOREIGN KEY (dqstats_id) REFERENCES dqstatsRun (dqstats_id), + FOREIGN KEY (exp_id) REFERENCES rawExp (exp_id), + FOREIGN KEY (chip_id) REFERENCES chipRun (chip_id), + FOREIGN KEY (cam_id) REFERENCES camRun (cam_id), + FOREIGN KEY (warp_id) REFERENCES warpRun (warp_id) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; Index: branches/eam_branches/ipp-20130711/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/pxadmin_drop_tables.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/pxadmin_drop_tables.sql (revision 36071) @@ -118,4 +118,6 @@ DROP TABLE IF EXISTS mergedvodbProcessed; DROP TABLE IF EXISTS mergedvodbRun; +DROP TABLE IF EXISTS dqstatsContent; +DROP TABLE IF EXISTS dqstatsRun; SET FOREIGN_KEY_CHECKS=1 Index: branches/eam_branches/ipp-20130711/ippTools/share/regtool_pendingburntoolimfile.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/regtool_pendingburntoolimfile.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/regtool_pendingburntoolimfile.sql (revision 36071) @@ -12,4 +12,5 @@ newImfile.tmp_class_id, summitExp.exp_type, + rawImfile.camera AS camera, rawImfile.dateobs, rawImfile.obs_mode AS obs_mode, Index: branches/eam_branches/ipp-20130711/ippTools/share/releasetool_listrelexp.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/releasetool_listrelexp.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/releasetool_listrelexp.sql (revision 36071) @@ -7,4 +7,6 @@ ippRelease.rel_id, ippRelease.priority, + ippRelease.dvodb, + ippRelease.ubercal_file, survey.surveyName, rawExp.exp_name, @@ -12,4 +14,6 @@ relExp.zpt_obs, relExp.zpt_stdev, + relExp.mcal, + relExp.ubercal_dist, relExp.path_base, relExp.registered, Index: branches/eam_branches/ipp-20130711/ippTools/share/releasetool_stacksummary.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/releasetool_stacksummary.sql (revision 36071) +++ branches/eam_branches/ipp-20130711/ippTools/share/releasetool_stacksummary.sql (revision 36071) @@ -0,0 +1,19 @@ +SELECT DISTINCT sass_id, + stackAssociation.tess_id, + stackAssociation.projection_cell, + stackAssociation.filter, + stackSummary.path_base, + stackAssociation.data_group, + survey.surveyName, + ippRelease.release_name, + ippRelease.rel_id + +FROM relStack + JOIN ippRelease USING(rel_id) + JOIN survey USING(surveyID) + JOIN stackAssociationMap USING(stack_id) + JOIN stackAssociation USING(sass_id, filter) + JOIN stackSummary using(sass_id, projection_cell) + +-- WHERE rel_id = 37 and skycell_id like 'skycell.0635.045' and filter = 'r.00000'\G +-- WHERE rel_id = 37 and projection_cell = 'skycell.0635' and filter = 'r.00000'\G Index: branches/eam_branches/ipp-20130711/ippTools/share/releasetool_summary.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/releasetool_summary.sql (revision 36071) +++ branches/eam_branches/ipp-20130711/ippTools/share/releasetool_summary.sql (revision 36071) @@ -0,0 +1,23 @@ +SELECT DISTINCT + release_name, + surveyName, + sass_id, + stackAssociation.tess_id, + stackAssociation.projection_cell, + stackAssociation.filter, + stackSummary.path_base, + stackAssociation.data_group, + rawExp.camera +FROM relStack + JOIN ippRelease USING(rel_id) + JOIN survey USING(surveyID) + JOIN stackRun USING(stack_id, filter, tess_id, skycell_id) + JOIN stackAssociationMap USING(stack_id) + JOIN stackAssociation USING(sass_id, filter) + JOIN stackSummary using(sass_id, projection_cell) + JOIN stackInputSkyfile using(stack_id) + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) Index: branches/eam_branches/ipp-20130711/ippTools/share/stacktool_summary.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/stacktool_summary.sql (revision 36071) +++ branches/eam_branches/ipp-20130711/ippTools/share/stacktool_summary.sql (revision 36071) @@ -0,0 +1,17 @@ +SELECT DISTINCT sass_id, + stackAssociation.tess_id, + stackAssociation.projection_cell, + stackAssociation.filter, + stackSummary.path_base, + stackAssociation.data_group, + rawExp.camera +FROM stackRun + JOIN stackAssociationMap USING(stack_id) + JOIN stackAssociation USING(sass_id, filter) + JOIN stackSummary using(sass_id, projection_cell) + JOIN stackInputSkyfile using(stack_id) + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) Index: branches/eam_branches/ipp-20130711/ippTools/share/stacktool_tosummary.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/stacktool_tosummary.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/stacktool_tosummary.sql (revision 36071) @@ -1,3 +1,5 @@ -SELECT DISTINCT sass_id,rawExp.camera,stackRun.workdir,stackRun.tess_id,stackRun.state +SELECT sass_id,camera,workdir,tess_id,state FROM +(SELECT sass_id,rawExp.camera,stackRun.workdir,stackRun.tess_id,stackRun.state, + sum((stackRun.state = 'full')) AS is_done, sum(1) AS is_defined FROM stackRun JOIN stackInputSkyfile ON stackRun.stack_id = stackInputSkyfile.stack_id @@ -12,2 +14,6 @@ WHERE stackRun.state = 'full' AND stackSummary.projection_cell IS NULL +-- WHERE HOOK %s +GROUP BY sass_id +) AS T +WHERE is_done = is_defined Index: branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_export_skycalresult.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_export_skycalresult.sql (revision 36071) +++ branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_export_skycalresult.sql (revision 36071) @@ -0,0 +1,1 @@ +SELECT * from skycalResult Index: branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_export_skycalrun.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_export_skycalrun.sql (revision 36071) +++ branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_export_skycalrun.sql (revision 36071) @@ -0,0 +1,1 @@ +SELECT * from skycalRun Index: branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_skycalresult.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_skycalresult.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/staticskytool_skycalresult.sql (revision 36071) @@ -2,8 +2,10 @@ skycalResult.*, skycalRun.stack_id, + skycalRun.workdir, stackRun.filter, skycalRun.state, skycalRun.label, skycalRun.data_group, + skycalRun.dist_group, skycalRun.sky_id, skycell.* Index: branches/eam_branches/ipp-20130711/ippTools/share/warptool_scmap.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/warptool_scmap.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/warptool_scmap.sql (revision 36071) @@ -8,4 +8,5 @@ chipProcessedImfile.chip_id, chipRun.state, + chipRun.data_group as chip_data_group, chipProcessedImfile.data_state, chipProcessedImfile.fault AS chip_fault, Index: branches/eam_branches/ipp-20130711/ippTools/share/warptool_warped.sql =================================================================== --- branches/eam_branches/ipp-20130711/ippTools/share/warptool_warped.sql (revision 35801) +++ branches/eam_branches/ipp-20130711/ippTools/share/warptool_warped.sql (revision 36071) @@ -13,4 +13,5 @@ rawExp.decl, rawExp.exp_time, + camRun.cam_id, magicDSRun.state AS dsRun_state, IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id