Changeset 37068 for branches/eam_branches/ipp-ops-20130712/ippTools/share

Timestamp:

Jul 17, 2014, 12:36:19 PM (12 years ago)

Author:

eugene

Message:

merge changes (from past YEAR) from trunk

Location:

branches/eam_branches/ipp-ops-20130712

Files:

• . (modified) (1 prop)
• ippTools (modified) (1 prop)
• ippTools/share (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (5 diffs)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/camtool_find_processedexp.sql (modified) (1 diff)
• ippTools/share/camtool_pendingexp.sql (modified) (1 prop)
• ippTools/share/chiptool_pendingcleanuprun.sql (modified) (1 prop)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/detselect_show.sql (copied) (copied from trunk/ippTools/share/detselect_show.sql )
• ippTools/share/disttool_definebyquery_SSdiff.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_camera.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_chip.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_chip_bg.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_diff.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_fake.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_ff.sql (copied) (copied from trunk/ippTools/share/disttool_definebyquery_ff.sql )
• ippTools/share/disttool_definebyquery_raw.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_sky.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_sky_singlefilter.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_skycal.sql (modified) (2 diffs)
• ippTools/share/disttool_definebyquery_stack.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_warp.sql (modified) (1 diff)
• ippTools/share/disttool_definebyquery_warp_bg.sql (modified) (1 diff)
• ippTools/share/disttool_pending_ff.sql (copied) (copied from trunk/ippTools/share/disttool_pending_ff.sql )
• ippTools/share/disttool_toadvance.sql (modified) (1 diff)
• ippTools/share/fftool_definebyquery.sql (copied) (copied from trunk/ippTools/share/fftool_definebyquery.sql )
• ippTools/share/fftool_definebyquery_select_warps.sql (copied) (copied from trunk/ippTools/share/fftool_definebyquery_select_warps.sql )
• ippTools/share/fftool_export_input.sql (copied) (copied from trunk/ippTools/share/fftool_export_input.sql )
• ippTools/share/fftool_export_result.sql (copied) (copied from trunk/ippTools/share/fftool_export_result.sql )
• ippTools/share/fftool_export_run.sql (copied) (copied from trunk/ippTools/share/fftool_export_run.sql )
• ippTools/share/fftool_export_summary.sql (copied) (copied from trunk/ippTools/share/fftool_export_summary.sql )
• ippTools/share/fftool_result.sql (copied) (copied from trunk/ippTools/share/fftool_result.sql )
• ippTools/share/fftool_revert.sql (copied) (copied from trunk/ippTools/share/fftool_revert.sql )
• ippTools/share/fftool_revertsummary.sql (copied) (copied from trunk/ippTools/share/fftool_revertsummary.sql )
• ippTools/share/fftool_summary.sql (copied) (copied from trunk/ippTools/share/fftool_summary.sql )
• ippTools/share/fftool_summaryinputs.sql (copied) (copied from trunk/ippTools/share/fftool_summaryinputs.sql )
• ippTools/share/fftool_toadvance.sql (copied) (copied from trunk/ippTools/share/fftool_toadvance.sql )
• ippTools/share/fftool_todo.sql (copied) (copied from trunk/ippTools/share/fftool_todo.sql )
• ippTools/share/pstamptool_pendingdependent.sql (modified) (1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (2 diffs, 1 prop)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/share/releasetool_listrelexp.sql (modified) (1 prop)
• ippTools/share/releasetool_listrelstack.sql (modified) (1 prop)
• ippTools/share/remotetool_definebyquery_cam.sql (copied) (copied from trunk/ippTools/share/remotetool_definebyquery_cam.sql )
• ippTools/share/remotetool_definebyquery_chip.sql (copied) (copied from trunk/ippTools/share/remotetool_definebyquery_chip.sql )
• ippTools/share/remotetool_definebyquery_stack.sql (copied) (copied from trunk/ippTools/share/remotetool_definebyquery_stack.sql )
• ippTools/share/remotetool_definebyquery_warp.sql (copied) (copied from trunk/ippTools/share/remotetool_definebyquery_warp.sql )
• ippTools/share/remotetool_dropcomponent.sql (copied) (copied from trunk/ippTools/share/remotetool_dropcomponent.sql )
• ippTools/share/remotetool_updatepoll.sql (copied) (copied from trunk/ippTools/share/remotetool_updatepoll.sql )
• ippTools/share/stacktool_addsummary.sql (modified) (1 diff)
• ippTools/share/stacktool_revertsummary.sql (copied) (copied from trunk/ippTools/share/stacktool_revertsummary.sql )
• ippTools/share/stacktool_tosummary.sql (modified) (1 prop)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/share/warptool_warped.sql (modified) (1 prop)

branches/eam_branches/ipp-ops-20130712/ippTools/share/Makefile.am

@@ -105 +105 @@
         detselect_search.sql \
         detselect_select.sql \
+        detselect_show.sql \
         dettool_addprocessedexp.sql \
         dettool_childlessrun.sql \
@@ -189 +190 @@
         disttool_definebyquery_diff.sql \
         disttool_definebyquery_fake.sql \
+        disttool_definebyquery_ff.sql \
         disttool_definebyquery_raw.sql \
         disttool_definebyquery_raw_no_magic.sql \
@@ -206 +208 @@
         disttool_pending_diff.sql \
         disttool_pending_fake.sql \
+        disttool_pending_ff.sql \
         disttool_pending_raw.sql \
         disttool_pending_sky.sql \
@@ -400 +403 @@
         stacktool_tosummary.sql \
         stacktool_addsummary.sql \
+        stacktool_revertsummary.sql \
         stacktool_summary.sql \
         staticskytool_definebyquery_select.sql \
@@ -487 +491 @@
         releasetool_definerelgroup_select_exp_lap.sql \
         releasetool_stacksummary.sql \
-        releasetool_pendingrelgroup.sql
-
+        releasetool_pendingrelgroup.sql \
+        fftool_definebyquery.sql \
+        fftool_definebyquery_select_warps.sql \
+        fftool_result.sql \
+        fftool_revert.sql \
+        fftool_revertsummary.sql \
+        fftool_summary.sql \
+        fftool_todo.sql \
+        fftool_toadvance.sql \
+        fftool_export_run.sql \
+        fftool_export_input.sql \
+        fftool_export_result.sql \
+        fftool_export_summary.sql \
+        remotetool_definebyquery_chip.sql \
+        remotetool_definebyquery_cam.sql \
+        remotetool_definebyquery_warp.sql \
+        remotetool_definebyquery_stack.sql \
+        remotetool_dropcomponent.sql \
+        remotetool_updatepoll.sql

branches/eam_branches/ipp-ops-20130712/ippTools/share/camtool_find_processedexp.sql

@@ -1 +1 @@
 SELECT
     camProcessedExp.*,
+    HEX(IFNULL(camProcessedExp.astrom_chips, 0)) AS good_astrom_chips,
     camRun.workdir,
     camRun.label,

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_SSdiff.sql

@@ -22 +22 @@
       AND distTarget.state = 'enabled'
       AND rcInterest.state = 'enabled'
-      AND distRun.dist_id IS NULL
-
       AND ((diffRun.state = 'full') OR (distTarget.clean AND diffRun.state = 'cleaned'))

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_camera.sql

@@ -21 +21 @@
 WHERE distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL      -- no existing distRun 
     AND ((camRun.state = 'full') OR (distTarget.clean AND camRun.state = 'cleaned'))

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_chip.sql

@@ -20 +20 @@
 WHERE distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL      -- no existing distRun 
     AND ((chipRun.state = 'full') OR (distTarget.clean AND chipRun.state = 'cleaned'))

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_chip_bg.sql

@@ -23 +23 @@
 WHERE distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL      -- no existing distRun
     AND ((chipBackgroundRun.state = 'full') OR (distTarget.clean AND chipBackgroundRun.state = 'cleaned'))

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_diff.sql

@@ -25 +25 @@
 WHERE  distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL
     AND ((diffRun.state = 'full') OR (distTarget.clean AND diffRun.state = 'cleaned'))
 

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_fake.sql

@@ -20 +20 @@
 WHERE  distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL
     AND ((fakeRun.state = 'full') OR (distTarget.clean AND fakeRun.state = 'cleaned'))

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_raw.sql

@@ -16 +16 @@
 JOIN rcInterest USING(target_id)
 LEFT JOIN distRun ON distRun.stage_id = exp_id AND distRun.target_id = distTarget.target_id
-WHERE distRun.dist_id IS NULL           -- no existing distRun for this exposure
-    AND distTarget.state = 'enabled'    -- target and intrest are enabled
+WHERE distTarget.state = 'enabled'    -- target and intrest are enabled
     AND rcInterest.state = 'enabled'
     AND magicDSRun.state = 'full'       -- destreaked files are available

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_sky.sql

@@ -23 +23 @@
     AND rcInterest.state = 'enabled'
     AND distTarget.filter = 'multi'
-    AND distRun.dist_id IS NULL
     AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned'))
     -- we shouldn't need to check fault. If faulted it shouldn't be full

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_sky_singlefilter.sql

@@ -22 +22 @@
 WHERE  distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL
     AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned'))
     AND stackRun.filter = distTarget.filter

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_skycal.sql

@@ -1 @@
-SELECT
+SELECT DISTINCT
     'skycal' as stage,
     skycalRun.skycal_id AS stage_id,
@@ -21 +21 @@
 WHERE  distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL
     AND ((skycalRun.state = 'full') OR (distTarget.clean AND skycalRun.state = 'cleaned'))
     AND stackRun.filter = distTarget.filter

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_stack.sql

@@ -21 +21 @@
 WHERE  distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL
     AND ((stackRun.state = 'full') OR (distTarget.clean AND stackRun.state = 'cleaned'))
     -- we shouldn't need to check fault. If faulted it shouldn't be full

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_warp.sql

@@ -22 +22 @@
 WHERE  distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL
     AND ((warpRun.state = 'full') OR (distTarget.clean AND warpRun.state = 'cleaned'))
 

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_definebyquery_warp_bg.sql

@@ -24 +24 @@
 WHERE  distTarget.state = 'enabled'
     AND rcInterest.state = 'enabled'
-    AND distRun.dist_id IS NULL
     AND ((warpBackgroundRun.state = 'full') OR (distTarget.clean AND warpBackgroundRun.state = 'cleaned'))
 

branches/eam_branches/ipp-ops-20130712/ippTools/share/disttool_toadvance.sql

@@ -308 +308 @@
         COUNT(diffSkyfile.skycell_id) = COUNT(distComponent.component)
         AND SUM(distComponent.fault) = 0
+UNION
+-- ff stage
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    distRun.label,
+    clean
+    -- ,COUNT(warp_id)
+    -- ,COUNT(ff_id)
+    FROM distRun
+    JOIN fullForceRun ON stage_id = ff_id
+    JOIN fullForceResult USING(ff_id)
+    JOIN skycalRun USING(skycal_id)
+    JOIN stackRun USING(stack_id)
+    LEFT JOIN distComponent
+        ON distRun.dist_id = distComponent.dist_id
+        AND distComponent.component = CONCAT_WS('.', warp_id, stackRun.skycell_id) 
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distRun.stage = 'ff'
+    GROUP BY
+        distRun.dist_id,
+        ff_id
+    HAVING
+        -- number of dist components is the number of warps plus 1 (for the summary component)
+        COUNT(fullForceResult.ff_id) = COUNT(distComponent.component)
+        AND SUM(distComponent.fault) = 0
 ) as Foo

branches/eam_branches/ipp-ops-20130712/ippTools/share/pxadmin_create_tables.sql

@@ -523 +523 @@
     quality SMALLINT NOT NULL DEFAULT 0,
     background_model SMALLINT,
+    astrom_chips BIGINT,
     PRIMARY KEY(cam_id),
     KEY(fault),
@@ -2307 +2308 @@
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 
+CREATE TABLE fullForceRun (
+    ff_id           BIGINT NOT NULL AUTO_INCREMENT,
+    skycal_id       BIGINT,
+    sources_path_base VARCHAR(255),
+    state           VARCHAR(64),
+    workdir         VARCHAR(255),
+    label           VARCHAR(64),
+    data_group      VARCHAR(64),
+    dist_group      VARCHAR(64),
+    note            VARCHAR(255),
+    reduction       VARCHAR(64),
+    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
+    PRIMARY KEY(ff_id),
+    KEY(state),
+    KEY(label),
+    KEY(data_group),
+    KEY(skycal_id),
+    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceInput (
+    ff_id           BIGINT,
+    warp_id         BIGINT,
+    PRIMARY KEY(ff_id, warp_id),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceResult (
+    ff_id           BIGINT,
+    warp_id         BIGINT,
+    path_base       VARCHAR(255) NOT NULL,
+    dtime_script    FLOAT,
+    quality         SMALLINT NOT NULL,
+    hostname        VARCHAR(64) NOT NULL,
+    software_ver    VARCHAR(16),
+    fault           SMALLINT NOT NULL,
+    PRIMARY KEY(ff_id, warp_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceSummary (
+    ff_id           BIGINT,
+    path_base       VARCHAR(255) NOT NULL,
+    dtime_script    FLOAT,
+    quality         SMALLINT NOT NULL,
+    hostname        VARCHAR(64) NOT NULL,
+    software_ver    VARCHAR(16),
+    fault           SMALLINT NOT NULL,
+    PRIMARY KEY(ff_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE remoteRun (
+    remote_id BIGINT NOT NULL AUTO_INCREMENT,
+    state     VARCHAR(64) NOT NULL,
+    stage     VARCHAR(64) NOT NULL,
+    label     VARCHAR(64) NOT NULL,
+    path_base VARCHAR(255) NOT NULL,
+    job_id    BIGINT,
+    last_poll TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    fault     SMALLINT NOT NULL,
+    PRIMARY KEY (remote_id),
+    KEY (state),
+    KEY (stage),
+    KEY (label),
+    KEY (job_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE remoteComponent (
+    remote_id BIGINT,
+    stage_id  BIGINT,
+    KEY (stage_id),
+    FOREIGN KEY (remote_id) REFERENCES remoteRun(remote_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 -- These comment lines are here to avoid an empty query error.

branches/eam_branches/ipp-ops-20130712/ippTools/share/pxadmin_drop_tables.sql

@@ -120 +120 @@
 DROP TABLE IF EXISTS dqstatsContent;
 DROP TABLE IF EXISTS dqstatsRun;
+DROP TABLE IF EXISTS fullForceRun;
+DROP TABLE IF EXISTS fullForceInput;
+DROP TABLE IF EXISTS fullForceResult;
+DROP TABLE IF EXISTS fullForceSummary;
+
 
 SET FOREIGN_KEY_CHECKS=1

branches/eam_branches/ipp-ops-20130712/ippTools/share/stacktool_addsummary.sql

@@ -1 +1 @@
 INSERT INTO stackSummary (sass_id,projection_cell,path_base) VALUES (
-       %lld,   -- warp_id
+       %lld,   -- sass_id
        '%s',   -- projection_cell
        '%s'    -- path_base