Index: branches/eam_branches/ipp-20110710/ippTools/share/Makefile.am
===================================================================
--- branches/eam_branches/ipp-20110710/ippTools/share/Makefile.am	(revision 31852)
+++ branches/eam_branches/ipp-20110710/ippTools/share/Makefile.am	(revision 32337)
@@ -244,4 +244,5 @@
 	magictool_definebyquery_insert.sql \
 	magictool_definebyquery_select.sql \
+	magictool_definebyquery_select_multidiff.sql \
 	magictool_inputs.sql \
 	magictool_inputskyfile.sql \
Index: branches/eam_branches/ipp-20110710/ippTools/share/dettool_toresidimfile.sql
===================================================================
--- branches/eam_branches/ipp-20110710/ippTools/share/dettool_toresidimfile.sql	(revision 31852)
+++ branches/eam_branches/ipp-20110710/ippTools/share/dettool_toresidimfile.sql	(revision 32337)
@@ -101,3 +101,49 @@
  AND detResidImfile.exp_id IS NULL
  AND detResidImfile.class_id IS NULL
+UNION
+SELECT
+    detRun.det_id,
+    detRun.iteration,
+    detRun.det_type,
+    detRun.mode,
+    detRun.workdir,
+    detRun.reduction,
+    detProcessedImfile.exp_id,
+    detProcessedImfile.class_id,
+    detProcessedImfile.uri,
+    detRegisteredImfile.uri AS det_uri,
+    detRun.ref_det_id,
+    detRun.ref_iter,
+    rawExp.camera,
+    rawExp.exp_tag
+FROM detRun
+JOIN detRun AS detRunRef
+  ON detRunRef.det_id = detRun.ref_det_id
+JOIN detInputExp
+  ON detInputExp.det_id    = detRun.det_id
+ AND detInputExp.iteration = detRun.iteration
+JOIN rawExp
+  ON rawExp.exp_id = detInputExp.exp_id
+JOIN detProcessedImfile
+  ON detProcessedImfile.det_id    = detRun.det_id
+ AND detProcessedImfile.exp_id    = detInputExp.exp_id
+JOIN detRegisteredImfile
+  ON detRegisteredImfile.det_id    = detRun.ref_det_id
+ AND detRegisteredImfile.iteration = detRun.ref_iter
+ AND detRegisteredImfile.class_id  = detProcessedImfile.class_id
+LEFT JOIN detResidImfile
+  ON detResidImfile.det_id    = detRun.det_id      
+ AND detResidImfile.iteration = detRun.iteration   
+ AND detResidImfile.exp_id    = detProcessedImfile.exp_id   
+ AND detResidImfile.class_id  = detProcessedImfile.class_id 
+WHERE
+ detRun.state = 'run'
+ AND detRun.mode = 'verify'
+ AND detRunRef.state = 'stop'
+ AND detRegisteredImfile.fault = 0
+ AND detProcessedImfile.fault = 0
+ AND detResidImfile.det_id IS NULL
+ AND detResidImfile.iteration IS NULL
+ AND detResidImfile.exp_id IS NULL
+ AND detResidImfile.class_id IS NULL
 ORDER BY exp_id
Index: branches/eam_branches/ipp-20110710/ippTools/share/difftool_pendingcleanuprun.sql
===================================================================
--- branches/eam_branches/ipp-20110710/ippTools/share/difftool_pendingcleanuprun.sql	(revision 31852)
+++ branches/eam_branches/ipp-20110710/ippTools/share/difftool_pendingcleanuprun.sql	(revision 32337)
@@ -2,5 +2,5 @@
 -- all of this is just to get the camera used for the diff run
 SELECT DISTINCT * FROM 
-   (SELECT 
+   (SELECT DISTINCT
 	    diffRun.diff_id,
 	    rawExp.camera,
@@ -29,6 +29,7 @@
              diffRun.state = 'goto_scrubbed' OR 
 	     diffRun.state = 'goto_purged')
+	@INNERLIMITS@
      UNION
-     SELECT
+     SELECT DISTINCT
 	    diffRun.diff_id,
 	    rawExp.camera,
@@ -57,6 +58,7 @@
              diffRun.state = 'goto_scrubbed' OR 
 	     diffRun.state = 'goto_purged')
+	@INNERLIMITS@
      UNION
-     SELECT
+     SELECT DISTINCT 
 	    diffRun.diff_id,
 	    rawExp.camera,
@@ -85,6 +87,7 @@
              diffRun.state = 'goto_scrubbed' OR 
 	     diffRun.state = 'goto_purged')
+	@INNERLIMITS@
      UNION
-     SELECT
+     SELECT DISTINCT
 	    diffRun.diff_id,
 	    rawExp.camera,
@@ -113,4 +116,5 @@
              diffRun.state = 'goto_scrubbed' OR 
 	     diffRun.state = 'goto_purged')
+	@INNERLIMITS@
 	) as Foo
 	WHERE 1
Index: branches/eam_branches/ipp-20110710/ippTools/share/dqstatstool_definebyquery.sql
===================================================================
--- branches/eam_branches/ipp-20110710/ippTools/share/dqstatstool_definebyquery.sql	(revision 31852)
+++ branches/eam_branches/ipp-20110710/ippTools/share/dqstatstool_definebyquery.sql	(revision 32337)
@@ -1,9 +1,10 @@
 -- query to identify unique data to bundle
 
-SELECT DISTINCT exp_id,chip_id,cam_id,warp_id,invalid from rawExp 
+SELECT DISTINCT exp_id,chip_id,cam_id,warpRun.warp_id,invalid from rawExp 
 	JOIN chipRun USING(exp_id) 
 	JOIN camRun USING(chip_id) 
+	JOIN camProcessedExp USING(cam_id)
 	LEFT OUTER JOIN fakeRun USING(cam_id) 
 	LEFT OUTER JOIN warpRun USING(fake_id)
-	LEFT OUTER JOIN dqstatsContent USING (exp_id,chip_id,cam_id,warp_id)
+	LEFT OUTER JOIN dqstatsContent USING (exp_id,chip_id,cam_id)
 WHERE camRun.state = 'full'
Index: branches/eam_branches/ipp-20110710/ippTools/share/magictool_definebyquery_select_multidiff.sql
===================================================================
--- branches/eam_branches/ipp-20110710/ippTools/share/magictool_definebyquery_select_multidiff.sql	(revision 32337)
+++ branches/eam_branches/ipp-20110710/ippTools/share/magictool_definebyquery_select_multidiff.sql	(revision 32337)
@@ -0,0 +1,53 @@
+-- Get a list of exposures on which magic may be performed
+SELECT DISTINCT
+    chipRun.exp_id,
+    MAX(diffWarps.diff_id) AS diff_id,
+    -- The following trick pulls out the 'inverse' value for the maximum diff_id
+    CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id DESC), ',', 1), UNSIGNED) AS inverse,
+    diff_data_group
+FROM (
+    -- Forward diffs
+    SELECT
+        diffRun.diff_id,
+        warp1 AS warp_id,
+        0 AS inverse,
+        diffRun.data_group AS diff_data_group
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    JOIN diffSkyfile USING(diff_id, skycell_id)
+    WHERE diffInputSkyfile.warp1 IS NOT NULL
+        AND diffRun.exposure = 1
+        AND diffRun.magicked = 0
+        AND diffSkyfile.quality = 0
+    -- diff WHERE hook %s
+    UNION
+    -- Backward diffs
+    SELECT
+        diffRun.diff_id,
+        warp2 AS warp_id,
+        1 AS inverse,
+        diffRun.data_group AS diff_data_group
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    JOIN diffSkyfile USING(diff_id, skycell_id)
+    WHERE diffInputSkyfile.warp2 IS NOT NULL
+        AND diffRun.exposure = 1
+        AND diffRun.bothways = 1
+        AND diffRun.magicked = 0
+        AND diffSkyfile.quality = 0
+    -- diff WHERE hook %s
+    ) AS diffWarps
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+LEFT JOIN
+    (SELECT magic_id, exp_id, diff_id, label
+        FROM magicRun
+        -- rerun hook %s
+    ) AS oldMagicRun
+    ON oldMagicRun.exp_id = chipRun.exp_id
+    AND oldMagicRun.diff_id = diffWarps.diff_id
+-- WHERE hook %s
+GROUP BY chipRun.exp_id
Index: branches/eam_branches/ipp-20110710/ippTools/share/pubtool_definerun.sql
===================================================================
--- branches/eam_branches/ipp-20110710/ippTools/share/pubtool_definerun.sql	(revision 31852)
+++ branches/eam_branches/ipp-20110710/ippTools/share/pubtool_definerun.sql	(revision 32337)
@@ -1,3 +1,3 @@
--- Get runs to publish
+-- Get runs to publish 
 SELECT DISTINCT
     client_id,
@@ -20,8 +20,14 @@
     JOIN chipRun USING(chip_id)
     JOIN rawExp USING(exp_id)
+    LEFT JOIN magicDSRun ON magicDSRun.stage = 'diff' AND magicDSRun.stage_id = diff_id
     WHERE publishClient.stage = 'diff'
         AND publishClient.active = 1
         AND diffRun.state IN ('full', 'cleaned', 'goto_cleaned')
-        AND (diffRun.magicked != 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0)
+        AND ( diffRun.diff_mode = 4
+           OR (publishClient.magicked AND diffRun.magicked != 0)
+           OR (publishClient.magicked = 0 AND 
+                (diffRun.magicked = 0 OR (diffRun.magicked != 0 AND magic_ds_id IS NOT NULL))
+              )
+           )
     -- WHERE hook %s
     UNION
Index: branches/eam_branches/ipp-20110710/ippTools/share/pxadmin_create_tables.sql
===================================================================
--- branches/eam_branches/ipp-20110710/ippTools/share/pxadmin_create_tables.sql	(revision 31852)
+++ branches/eam_branches/ipp-20110710/ippTools/share/pxadmin_create_tables.sql	(revision 32337)
@@ -533,5 +533,6 @@
 CREATE TABLE addRun (
     add_id BIGINT AUTO_INCREMENT,
-    cam_id BIGINT,
+    stage VARCHAR(64),
+    stage_id BIGINT,
     state VARCHAR(64),
     workdir VARCHAR(255),
@@ -543,13 +544,16 @@
     note  VARCHAR(255),
     image_only TINYINT,
+    minidvodb TINYINT, 
+    minidvodb_group VARCHAR(64),
+    minidvodb_name VARCHAR(64),
     PRIMARY KEY(add_id),
     KEY(add_id),
-    KEY(cam_id),
+    KEY(stage_id),
+    KEY(stage),
     KEY(state),
     KEY(workdir_state),
     KEY(label),
     KEY(data_group),
-    INDEX(add_id, cam_id),
-    FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
+    INDEX(add_id, cam_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -558,4 +562,5 @@
     dtime_addstar FLOAT,
     path_base VARCHAR(255),
+    dvodb_path VARCHAR(255),
     fault SMALLINT NOT NULL,
     PRIMARY KEY(add_id),
@@ -1946,4 +1951,47 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;    
 
+CREATE TABLE minidvodbRun (
+minidvodb_id BIGINT AUTO_INCREMENT,
+       minidvodb_name VARCHAR(64),
+       minidvodb_group VARCHAR(64) NOT NULL,
+       minidvodb_path VARCHAR(255) NOT NULL,
+       mergedvodb_path VARCHAR(255) NOT NULL,
+       state VARCHAR(64) NOT NULL,
+       creation_date TIMESTAMP,
+       PRIMARY KEY(minidvodb_id), KEY(minidvodb_name), INDEX(minidvodb_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE minidvodbProcessed (
+      minidvodb_id BIGINT,
+      merge_order BIGINT,
+      dtime_resort FLOAT,
+      dtime_relphot FLOAT,
+      dtime_verify FLOAT,
+      dtime_merge FLOAT,
+      dtime_script,
+      epoch TIMESTAMP,
+      mergedvodb_path VARCHAR(255) NOT NULL,
+      fault SMALLINT NOT NULL,
+       KEY(minidvodb_id), CONSTRAINT UNIQUE(minidvodb_id), FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun (minidvodb_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
+CREATE TABLE minidvodbCopy (
+    minidvodbcopy_id     BIGINT AUTO_INCREMENT,
+    minidvodb_id     BIGINT,
+    minidvodb_rsync_path VARCHAR(255) NOT NULL,
+    destination_host VARCHAR(255) NOT NULL,
+    fault  SMALLINT NOT NULL,
+    state   VARCHAR(64) NOT NULL,
+    epoch TIMESTAMP,
+    dtime FLOAT,
+    PRIMARY KEY(minidvodbcopy_id),
+    KEY(minidvodb_id),
+    FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
+
+
 -- This comment line is here to avoid empty query error.
 -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy.