Changeset 35236 for branches/eam_branches/ipp-20130207/ippTools/share

Timestamp:

Mar 5, 2013, 2:24:05 PM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130207

Files:

• . (modified) (1 prop)
• ippTools/share (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (2 diffs)
• ippTools/share/bgtool_chip.sql (modified) (2 diffs)
• ippTools/share/bgtool_chipinputs.sql (modified) (1 diff)
• ippTools/share/bgtool_definechip.sql (modified) (1 diff)
• ippTools/share/bgtool_definewarp.sql (modified) (1 diff)
• ippTools/share/bgtool_tochip.sql (modified) (1 diff)
• ippTools/share/bgtool_towarp.sql (modified) (1 diff)
• ippTools/share/bgtool_warp.sql (modified) (1 diff)
• ippTools/share/bgtool_warpinputs.sql (modified) (1 diff)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/difftool_definewarpstack.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (9 diffs, 1 prop)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/share/releasetool_definerelexp.sql (modified) (1 diff)
• ippTools/share/releasetool_definerelgroup_select_data_group.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelgroup_select_data_group.sql )
• ippTools/share/releasetool_definerelgroup_select_exp_data_group.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelgroup_select_exp_data_group.sql )
• ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql )
• ippTools/share/releasetool_definerelgroup_select_lap.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelgroup_select_lap.sql )
• ippTools/share/releasetool_definerelstack.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelstack.sql )
• ippTools/share/releasetool_definerelstack_with_skycal.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelstack_with_skycal.sql )
• ippTools/share/releasetool_listrelexp.sql (modified) (2 diffs)
• ippTools/share/releasetool_pendingrelgroup.sql (copied) (copied from trunk/ippTools/share/releasetool_pendingrelgroup.sql )
• ippTools/share/releasetool_tocalibexp.sql (copied) (copied from trunk/ippTools/share/releasetool_tocalibexp.sql )
• ippTools/share/reltool_pendinggroup.sql (copied) (copied from trunk/ippTools/share/reltool_pendinggroup.sql )
• ippTools/share/warptool_scmap.sql (modified) (1 diff)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)

branches/eam_branches/ipp-20130207/ippTools/share/Makefile.am

@@ -155 +155 @@
         difftool_completed_runs.sql \
         difftool_coalesce_run.sql \
+        difftool_definewarpstack.sql \
         difftool_definewarpstack_part1.sql \
         difftool_definewarpstack_part2.sql \
@@ -463 +464 @@
         sctool_list.sql \
         releasetool_listsurvey.sql \
+        releasetool_tocalibexp.sql \
         releasetool_listrelease.sql \
         releasetool_listrelexp.sql \
-        releasetool_definerelexp.sql
+        releasetool_definerelexp.sql \
+        releasetool_definerelstack.sql \
+        releasetool_definerelstack_with_skycal.sql \
+        releasetool_definerelgroup_select_lap.sql \
+        releasetool_definerelgroup_select_data_group.sql \
+        releasetool_definerelgroup_select_exp_data_group.sql \
+        releasetool_definerelgroup_select_exp_lap.sql \
+        releasetool_pendingrelgroup.sql
 

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chip.sql

@@ -5 +5 @@
     chipBackgroundRun.label,
     chipBackgroundRun.data_group,
+    chipBackgroundRun.chip_id,
+    chipBackgroundRun.cam_id,
     rawExp.exp_id,
     rawExp.exp_name,
@@ -12 +14 @@
     rawExp.ra,
     rawExp.decl,
-    rawExp.exp_time,
-    chipRun.chip_id
+    rawExp.exp_time
 FROM chipBackgroundRun
 JOIN chipBackgroundImfile USING(chip_bg_id)

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chipinputs.sql

@@ -1 +1 @@
 SELECT
-    chipProcessedImfile.*
+    chipProcessedImfile.*,
+    camProcessedExp.path_base AS cam_path_base
 FROM chipBackgroundRun
 JOIN chipRun USING(chip_id)
 JOIN chipProcessedImfile USING(chip_id)
+LEFT JOIN camProcessedExp ON chipBackgroundRun.cam_id = camProcessedExp.cam_id
 WHERE chipRun.state = 'full'
     AND chipProcessedImfile.fault = 0

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definechip.sql

@@ -1 +1 @@
 SELECT
-    chipRun.*
+    chipRun.*,
+    camRun.cam_id,
+    chip_bg_id,
+    CURRENT_TIMESTAMP AS registered
 FROM chipRun
 JOIN rawExp USING(exp_id)
-LEFT JOIN chipBackgroundRun USING(chip_id)
-WHERE chipRun.state = 'full'
+JOIN chipRun as altChipRun USING(exp_id)
+JOIN camRun ON altChipRun.chip_id = camRun.chip_id
+JOIN camProcessedExp USING(cam_id)
+LEFT JOIN chipBackgroundRun ON chipBackgroundRun.chip_id = chipRun.chip_id -- labelHook %s
+WHERE chipRun.state = 'full' AND camRun.state ='full' AND camProcessedExp.quality = 0

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definewarp.sql

@@ -1 +1 @@
 SELECT
-    warpRun.*,
-    chipBackgroundRun.chip_bg_id
+    warpRun.warp_id,
+    chipBackgroundRun.*
+    -- the following items are selected for aid in debugging
+    ,
+    warpsChipRun.chip_id as warpschip_id,
+    chipRun.chip_id AS thischip_id,
+    warpBackgroundRun.warp_bg_id
 FROM warpRun
 JOIN fakeRun USING(fake_id)
 JOIN camRun USING(cam_id)
-JOIN chipRun USING(chip_id)
-JOIN chipBackgroundRun USING(chip_id)
+JOIN chipRun as warpsChipRun ON camRun.chip_id = warpsChipRun.chip_id
 JOIN rawExp USING(exp_id)
-LEFT JOIN warpBackgroundRun USING(chip_bg_id)
+JOIN chipRun USING(exp_id)
+JOIN chipBackgroundRun ON chipRun.chip_id = chipBackgroundRun.chip_id 
+    AND (chipBackgroundRun.cam_id = camRun.cam_id OR chipBackgroundRun.cam_id = 0)
+LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.warp_bg_id -- label hook %s
 WHERE chipBackgroundRun.state = 'full'
-    AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed
+    AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- need warp to have completed so warpSkyCellMap is populated

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_tochip.sql

@@ -16 +16 @@
     AND (Label.active OR Label.active IS NULL)
 -- WHERE hook %s
-ORDER BY priority DESC, chip_bg_id
+ORDER BY priority DESC, chip_bg_id, class_id

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_towarp.sql

@@ -24 +24 @@
     AND (Label.active OR Label.active IS NULL)
 -- WHERE hook %s
-ORDER BY priority DESC, chip_bg_id
+ORDER BY priority DESC, warp_bg_id, skycell_id

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warp.sql

@@ -2 +2 @@
     warpBackgroundSkyfile.*,
     warpBackgroundRun.state,
+    warpBackgroundRun.warp_id,
+    camRun.cam_id,
+    warpBackgroundRun.chip_bg_id,
     warpBackgroundRun.workdir,
     warpBackgroundRun.label,
+    warpRun.label as warp_label,
     rawExp.exp_id,
     rawExp.exp_name,

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warpinputs.sql

@@ -11 +11 @@
 JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
 JOIN fakeRun USING(fake_id)
-JOIN camProcessedExp USING(cam_id)
+JOIN camProcessedExp ON camProcessedExp.cam_id = fakeRun.cam_id
 WHERE warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed
     AND warpSkyfile.fault = 0

branches/eam_branches/ipp-20130207/ippTools/share/difftool_definewarpstack.sql

@@ -1 @@
--- Get list of warps that can be diffed, with any associated diff,
--- and the best stack to use as a template
--- Warps without an existing diff can be identified by a NULL diff_id
+-- Get list of warp-stack pairs that can be diffed
+-- and check results against existing diffs
 SELECT
-    warpsToDiff.warp_id,
-    warpsToDiff.skycell_id,
-    warpsToDiff.tess_id,
-    warpsToDiff.filter,
-    warpsToDiff.good_frac,
-    warpsToDiff.diff_id,
-    current_stack_id,
-    best_stack_id,
-    exp_id
-FROM (
-    -- Get list of warps that can be diffed, with any associated diff
-    SELECT
-        warpSkyfile.warp_id,
-        warpSkyfile.skycell_id,
-        warpSkyfile.tess_id,
-        warpSkyfile.good_frac,
-        warpRun.label,
-        filter,
-        warpRun.label as warp_label,
-        diffInputs.diff_id,
-        diffInputs.stack2 AS current_stack_id,
-        rawExp.exp_id
-    FROM warpSkyfile
-    JOIN warpRun USING(warp_id)
-    JOIN fakeRun USING(fake_id)
-    JOIN camRun USING(cam_id)
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    -- Check if it has an associated diff
-    LEFT JOIN diffInputSkyfile AS diffInputs
-        ON diffInputs.warp1 = warpSkyfile.warp_id
-        AND diffInputs.skycell_id = warpSkyfile.skycell_id
-        AND diffInputs.stack2 IS NOT NULL
-    WHERE
-        warpSkyfile.fault = 0
-        AND warpSkyfile.quality = 0
-    -- warpsToDiff WHERE hook %s
-    ) AS warpsToDiff
--- Get best stack as a function of skycell_id, filter
-JOIN (
-    SELECT
-        MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment
-        skycell_id,
-        filter
-    FROM stackRun
-    JOIN stackSumSkyfile USING(stack_id)
-    WHERE stackRun.state = 'full'
-        AND stackSumSkyfile.fault = 0
-        AND stackSumSkyfile.quality = 0
-    -- stacksForDiff WHERE hook %s
-    GROUP BY
-        skycell_id,
-        filter
-    ) AS stacksForDiff USING(skycell_id, filter)
+   exp_id,
+   warp_id,
+   rawExp.filter,
+   warpRun.label AS warpLabel,
+   warpRun.data_group AS warpDataGroup,
+   warpRun.tess_id,
+   warpSkyfile.skycell_id,
+   MAX(stack_id) AS stack_id,
+   stackRun.label AS stackLabel,
+   stackRun.data_group AS stackDataGroup,
+   diff_id,
+   diffRun.label AS diffLabel
+FROM warpRun
+     JOIN fakeRun USING(fake_id)
+     JOIN camRun USING(cam_id)
+     JOIN chipRun USING(chip_id)
+     JOIN rawExp USING(exp_id)
+     JOIN warpSkyfile USING(warp_id)
+     JOIN stackRun ON 
+       (stackRun.skycell_id = warpSkyfile.skycell_id AND
+        stackRun.filter     = rawExp.filter AND
+        stackRun.tess_id    = warpRun.tess_id)
+     JOIN stackSumSkyfile USING(stack_id)
+     LEFT JOIN diffInputSkyfile ON 
+       (warp_id = diffInputSkyfile.warp1 AND
+        stack_id = diffInputSkyfile.stack2 AND
+        diffInputSkyfile.skycell_id = stackRun.skycell_id AND
+        diffInputSkyfile.tess_id = stackRun.tess_id)
+     LEFT JOIN diffRun USING(diff_id)
+WHERE 
+     warpRun.state = 'full'
+     AND stackRun.state = 'full'
+     AND warpSkyfile.fault = 0
+     AND warpSkyfile.quality = 0
+     AND stackSumSkyfile.fault = 0
+     AND stackSumSkyfile.quality = 0
+     AND exp_id IS NOT NULL
+-- %s
+GROUP BY exp_id,warp_id,skycell_id

branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_create_tables.sql

@@ -521 +521 @@
     deteff_uq FLOAT,
     quality SMALLINT NOT NULL DEFAULT 0,
+    background_model SMALLINT,
     PRIMARY KEY(cam_id),
     KEY(fault),
@@ -1791 +1792 @@
     chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier
     chip_id BIGINT NOT NULL,          -- link to chipRun
+    cam_id BIGINT DEFAULT 0,          -- optional link to camRun
     state VARCHAR(64) NOT NULL,       -- state of run (new, full, etc.)
     workdir VARCHAR(255) NOT NULL,    -- working directory
@@ -1843 +1845 @@
     data_group VARCHAR(64),           -- group for data
     dist_group VARCHAR(64),           -- group for distribution
-    reduction VARCHAR(64),    -- reduction class (for altering recipe)
+    reduction VARCHAR(64),            -- reduction class (for altering recipe)
     note VARCHAR(255),        -- note
     registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
@@ -2122 +2124 @@
       zpt_obs FLOAT,
       zpt_stdev FLOAT,
+      fwhm_major  FLOAT,
+      fwhm_minor  FLOAT,
       quality SMALLINT NOT NULL,
       hostname VARCHAR(64) NOT NULL,
@@ -2149 +2153 @@
 
 CREATE TABLE survey (
-    surveyID    INT(8),
+    surveyID    INT,
     surveyName  VARCHAR(16) NOT NULL,
     description VARCHAR(255),
     PRIMARY KEY(surveyID),
-    UNIQUE  KEY(name)
+    UNIQUE  KEY(surveyName)
 ) ENGINE=InnoDB CHARSET=latin1;
 
-
 CREATE TABLE ippRelease (
-    rel_id      INT(8) AUTO_INCREMENT,
-    surveyID    INT(8),
-    releaseName VARCHAR(64),
-    relase_state VARCHAR(16),   -- active, pending, archive, drop 
-    dataRelease INT(8),         -- PSPS dataRelease
-    priority    INT(8),
+    rel_id      INT AUTO_INCREMENT,
+    surveyID    INT,
+    release_name VARCHAR(64),
+    release_state VARCHAR(16),   -- active, pending, archive, drop 
+    dataRelease INT,         -- PSPS dataRelease
+    priority    INT,
     PRIMARY KEY(rel_id),
-    KEY(releaseName),
-    KEY(state),
+    KEY(release_name),
+    KEY(release_state),
     KEY(priority),
     FOREIGN KEY(surveyID) REFERENCES survey(surveyID)
@@ -2173 +2176 @@
 CREATE TABLE relExp (
     relexp_id   BIGINT AUTO_INCREMENT,
-    rel_id      INT(8),
+    rel_id      INT,
     exp_id      BIGINT,
     chip_id     BIGINT,         -- links to the runs that supplied the data
     cam_id      BIGINT,         -- for this release
+    group_id    INT,         -- id of relGroup that this exposure is contained in
     state       VARCHAR(16),    -- released, pending, archive, drop  
-    flags       INT UNSIGNED,   -- flags for relphot, relastro, ??
+    flags       INT UNSIGNED,  -- flags for relphot, relastro, ??
     zpt_obs     FLOAT,          -- calibrated zero point for this release of
     zpt_stdev   FLOAT,          -- this exposure
     path_base   VARCHAR(255),   -- path_base of any supporting files for this
                                 -- release of this exposure.
+    fault SMALLINT NOT NULL,
     registered  DATETIME,       -- insertion time for this row
     time_stamp  DATETIME,       -- time of last update for this row
@@ -2188 +2193 @@
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
     KEY (state),
+    KEY (fault),
     FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id),
     FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id),
@@ -2195 +2201 @@
 CREATE TABLE relStack (
     relstack_id BIGINT AUTO_INCREMENT,
-    rel_id      INT(8),
+    rel_id      INT,
+    stack_id    BIGINT,         -- id of the stackRun
     skycal_id   BIGINT,         -- id of the sky calibration run that supplied the calibration
-                                -- for this release. We get back to stack from here
-    skycell_id  varchar(64),
-    tess_id     varchar(64),
+                                -- for this release of this skycell.
+    skycell_id  VARCHAR(64),
+    tess_id     VARCHAR(64),
+    filter      VARCHAR(16),
     state       VARCHAR(16),    -- released, pending, archive, drop  
     flags       INT UNSIGNED,
@@ -2205 +2213 @@
     zpt_obs     FLOAT,          -- calibrated zero point for this release of this skycell
     zpt_stdev   FLOAT,
-    fwhm_major  FLOAT,          -- seeing as measured by staticsky
+    mjd_obs     INT UNSIGNED,   -- for nightly stacks the mjd_obs, zero for deep and reference
     path_base   VARCHAR(255),   -- path_base of any supporting files,
+    fault SMALLINT NOT NULL,
     registered  DATETIME,
     time_stamp  DATETIME,
     PRIMARY KEY (relstack_id),
     KEY (tess_id, skycell_id),
-    KEY (rel_id, tess_id, skycell_id),
+    KEY (rel_id, tess_id, skycell_id, filter, mjd_obs),
+    KEY (stack_type),
+    KEY (mjd_obs),
+    KEY (state),
+    KEY (fault),
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
-    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
+    FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE relGroup (
+    group_id    INT AUTO_INCREMENT,
+    rel_id      INT,
+    group_type  VARCHAR(16),
+    lap_id      BIGINT,
+    group_name  VARCHAR(16),
+    state       VARCHAR(16),    -- new full
+    label       VARCHAR(64),
+    exp_list_path   VARCHAR(255),
+    fault SMALLINT NOT NULL,
+    registered  DATETIME,
+    PRIMARY KEY (group_id),
+    KEY (rel_id, group_name),
+    KEY (group_type),
+    KEY (lap_id),
+    KEY (group_name),
+    KEY (state),
+    KEY (label),
+    KEY (fault),
+    FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE lapGroup  (
+    seq_id  BIGINT,
+    projection_cell VARCHAR(64),
+    state   VARCHAR(16),
+    fault   SMALLINT,
+    PRIMARY KEY(seq_id, projection_cell),
+    KEY(state),
+    KEY(fault),
+    FOREIGN KEY(seq_id) REFERENCES lapSequence(seq_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 

branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_drop_tables.sql

@@ -109 +109 @@
 DROP TABLE IF EXISTS skycalResult;
 DROP TABLE IF EXISTS skycell;
+DROP TABLE IF EXISTS relStack;
 DROP TABLE IF EXISTS relExp;
+DROP TABLE IF EXISTS relGroup;
 DROP TABLE IF EXISTS ippRelease;
 DROP TABLE IF EXISTS survey;
+DROP TABLE IF EXISTS lapGroup;
 
 SET FOREIGN_KEY_CHECKS=1

branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelexp.sql

@@ -1 +1 @@
 SELECT
     ippRelease.rel_id,
-    ippRelease.releaseName,
+    ippRelease.release_name,
     rawExp.exp_name,
     rawExp.filter,

branches/eam_branches/ipp-20130207/ippTools/share/releasetool_listrelexp.sql

@@ -2 +2 @@
     relexp_id,
     relExp.state,
-    ippRelease.releaseName,
+    relExp.fault,
+    ippRelease.release_name,
     ippRelease.release_state,
     ippRelease.rel_id,
@@ -21 +22 @@
     camProcessedExp.path_base as cam_path_base,
     camProcessedExp.fwhm_major,
+    camProcessedExp.fwhm_minor,
     warpRun.warp_id,
     warpRun.state as warp_state,

branches/eam_branches/ipp-20130207/ippTools/share/warptool_scmap.sql

@@ -5 +5 @@
     camProcessedExp.path_base as cam_path_base,
     camProcessedExp.fault as cam_fault,
+    camProcessedExp.background_model AS cam_background_model,
     chipProcessedImfile.chip_id,
     chipRun.state,