Index: trunk/dbconfig/changes.txt
===================================================================
--- trunk/dbconfig/changes.txt	(revision 24551)
+++ trunk/dbconfig/changes.txt	(revision 24552)
@@ -1164,2 +1164,34 @@
 
 -- Version 1.1.54
+
+-- changed magicked from a flag to a BIGINT
+-- The value is the magic_ds_id of the magicDSRun that destreaked the file/run
+
+ALTER TABLE magicDSFile ADD COLUMN data_state VARCHAR(64);
+ALTER TABLE magicDSFile ADD KEY(data_state);
+UPDATE magicDSFile SET data_state = 'full' WHERE data_state IS NULL;
+
+ALTER TABLE rawImfile CHANGE COLUMN magicked  magicked BIGINT;
+ALTER TABLE rawExp CHANGE COLUMN magicked  magicked BIGINT;
+
+ALTER TABLE chipProcessedImfile CHANGE COLUMN magicked  magicked BIGINT;
+ALTER TABLE chipRun CHANGE COLUMN magicked  magicked BIGINT;
+
+-- so far only chipRuns have been magicked in product
+UPDATE chipProcessedImfile JOIN magicDSRun ON stage = 'chip' AND chip_id = stage_id
+    SET chipProcessedImfile.magicked = magic_ds_id 
+    WHERE magicDSRun.re_place
+    AND chipProcessedImfile.magicked > 0;
+
+UPDATE chipRun JOIN magicDSRun ON stage = 'chip' AND chip_id = stage_id
+    SET chipRun.magicked = magic_ds_id 
+    WHERE magicDSRun.re_place
+    AND chipRun.magicked > 0;
+
+ALTER TABLE warpSkyfile CHANGE COLUMN magicked  magicked BIGINT;
+ALTER TABLE warpRun CHANGE COLUMN magicked  magicked BIGINT;
+
+ALTER TABLE diffSkyfile CHANGE COLUMN magicked  magicked BIGINT;
+ALTER TABLE diffRun CHANGE COLUMN magicked  magicked BIGINT;
+
+
Index: trunk/dbconfig/chip.md
===================================================================
--- trunk/dbconfig/chip.md	(revision 24551)
+++ trunk/dbconfig/chip.md	(revision 24552)
@@ -11,5 +11,5 @@
     tess_id     STR         64
     end_stage   STR         64      # Key
-    magicked	BOOL        f
+    magicked    S64         0
 END
 
@@ -89,5 +89,5 @@
     fault           S16     0       # Key NOT NULL
     quality         S16     0
-    magicked        BOOL    f
+    magicked        S64     0
 END
 
Index: trunk/dbconfig/diff.md
===================================================================
--- trunk/dbconfig/diff.md	(revision 24551)
+++ trunk/dbconfig/diff.md	(revision 24552)
@@ -12,5 +12,5 @@
     bothways    BOOL	    f
     exposure	BOOL	    f
-    magicked	BOOL        f
+    magicked    S64         0
 END
 
@@ -57,4 +57,4 @@
     fault        S16        0       # Key
     quality      S16        0
-    magicked     BOOL       f
+    magicked    S64         0
 END
Index: trunk/dbconfig/magic.md
===================================================================
--- trunk/dbconfig/magic.md	(revision 24551)
+++ trunk/dbconfig/magic.md	(revision 24552)
@@ -64,3 +64,4 @@
     recovery_path_base STR        255
     fault       S16         0
+    data_state  STR        64     # Key
 END
Index: trunk/dbconfig/raw.md
===================================================================
--- trunk/dbconfig/raw.md	(revision 24551)
+++ trunk/dbconfig/raw.md	(revision 24552)
@@ -65,5 +65,5 @@
     fault       S16         0       # Key NOT NULL
     epoch       UTC         0001-01-01T00:00:00Z
-    magicked	BOOL        f
+    magicked    S64         0
 END
 
@@ -135,4 +135,4 @@
     quality     S16         0
     epoch       UTC         0001-01-01T00:00:00Z
-    magicked    BOOL        f
+    magicked    S64         0
 END
Index: trunk/dbconfig/warp.md
===================================================================
--- trunk/dbconfig/warp.md	(revision 24551)
+++ trunk/dbconfig/warp.md	(revision 24552)
@@ -22,5 +22,5 @@
     end_stage   STR         64      # Key
     registered  TAI         NULL
-    magicked	BOOL        f
+    magicked    S64         0
 END
 
@@ -58,5 +58,5 @@
     fault          S16      0       # Key
     quality        S16      0
-    magicked       BOOL     f
+    magicked       S64      0
 END
 
Index: trunk/ippTools/share/pxadmin_create_tables.sql
===================================================================
--- trunk/ippTools/share/pxadmin_create_tables.sql	(revision 24551)
+++ trunk/ippTools/share/pxadmin_create_tables.sql	(revision 24552)
@@ -163,5 +163,5 @@
     fault SMALLINT NOT NULL,
     epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-    magicked TINYINT,
+    magicked BIGINT,
     PRIMARY KEY(exp_id),
     KEY(exp_name),
@@ -236,5 +236,5 @@
     epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     raw_image_id BIGINT AUTO_INCREMENT,
-    magicked TINYINT,
+    magicked BIGINT,
     PRIMARY KEY(exp_id, class_id),
     KEY(tmp_class_id),
@@ -259,5 +259,5 @@
     tess_id VARCHAR(64),
     end_stage VARCHAR(64),
-    magicked TINYINT,
+    magicked BIGINT,
     PRIMARY KEY(chip_id),
     KEY(chip_id), KEY(exp_id),
@@ -797,5 +797,5 @@
     end_stage VARCHAR(64),
     registered DATETIME,
-    magicked TINYINT,
+    magicked BIGINT,
     PRIMARY KEY(warp_id),
     KEY(warp_id),
@@ -850,5 +850,5 @@
     quality SMALLINT NOT NULL DEFAULT 0,
     fault SMALLINT,
-    magicked TINYINT,
+    magicked BIGINT,
     PRIMARY KEY(warp_id, skycell_id, tess_id),
     KEY(good_frac),
@@ -937,5 +937,5 @@
         bothways TINYINT DEFAULT 0,
         exposure TINYINT DEFAULT 0,
-        magicked TINYINT,
+        magicked BIGINT,
         PRIMARY KEY(diff_id),
         KEY(diff_id),
@@ -996,5 +996,5 @@
         quality SMALLINT NOT NULL DEFAULT 0,
         fault SMALLINT,
-        magicked TINYINT,
+        magicked BIGINT,
         PRIMARY KEY(diff_id, skycell_id),
         KEY(good_frac),
@@ -1093,6 +1093,8 @@
     recovery_path_base VARCHAR(255),
     fault SMALLINT,
+    data_state VARCHAR(64),
     PRIMARY KEY(magic_ds_id, component),
     KEY(fault),
+    KEY(data_state),
     FOREIGN KEY(magic_ds_id) REFERENCES magicDSRun(magic_ds_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
Index: trunk/ippTools/src/chiptool.c
===================================================================
--- trunk/ippTools/src/chiptool.c	(revision 24551)
+++ trunk/ippTools/src/chiptool.c	(revision 24552)
@@ -473,5 +473,5 @@
     PXOPT_LOOKUP_F32(n_cr, config->args,           "-n_cr", false, false);
     PXOPT_LOOKUP_STR(path_base, config->args,      "-path_base", false, false);
-    PXOPT_LOOKUP_BOOL(magicked, config->args,      "-magicked", false);
+    PXOPT_LOOKUP_S64(magicked, config->args,       "-magicked", false, false);
 
     // default values
@@ -601,5 +601,5 @@
     PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
     pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
-    PXOPT_COPY_S32(config->args, where, "-magicked", "chipRun.magicked", "==");
+    PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "==");
 
     if (!psListLength(where->list) &&
Index: trunk/ippTools/src/difftool.c
===================================================================
--- trunk/ippTools/src/difftool.c	(revision 24551)
+++ trunk/ippTools/src/difftool.c	(revision 24552)
@@ -49,5 +49,5 @@
 static bool importrunMode(pxConfig *config);
 
-static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked);
+static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked);
 static bool diffRunComplete(pxConfig *config);
 
@@ -512,5 +512,5 @@
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
-    PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false);
+    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
 
     if (!psDBTransaction(config->dbh)) {
@@ -739,5 +739,5 @@
 
 
-static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked)
+static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked)
 {
     PS_ASSERT_PTR_NON_NULL(state, false);
@@ -749,5 +749,5 @@
     }
 
-    char *query = "UPDATE diffRun SET state = '%s', magicked = %d WHERE diff_id = %"PRId64;
+    char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64;
 
     if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) {
@@ -803,5 +803,5 @@
             false,
             false,
-            false       // magicked
+            0       // magicked
     );
 
@@ -1190,5 +1190,5 @@
                 false,                  // bothways
                 true,                   // exposure
-                false       // magicked
+                0       // magicked
         );
 
@@ -1805,5 +1805,5 @@
 
         psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
-        bool magicked = psMetadataLookupBool(NULL, row, "magicked");
+        psS64 magicked = psMetadataLookupS64(NULL, row, "magicked");
 
         // set diffRun.state to 'stop'
Index: trunk/ippTools/src/magicdstool.c
===================================================================
--- trunk/ippTools/src/magicdstool.c	(revision 24551)
+++ trunk/ippTools/src/magicdstool.c	(revision 24552)
@@ -505,11 +505,11 @@
     // chose the appropriate query based on the stage
     if (!strcmp(stage, "raw")) {
-        query = "UPDATE rawImfile SET magicked = 1 where exp_id = %" PRId64 " AND class_id = '%s'";
+        query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'";
     } else if (!strcmp(stage, "chip")) {
-        query = "UPDATE chipProcessedImfile SET magicked = 1 where chip_id = %" PRId64 " AND class_id = '%s'";
+        query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'";
     } else if (!strcmp(stage, "warp")) {
-        query = "UPDATE warpSkyfile SET magicked = 1 where warp_id = %" PRId64 " AND skycell_id = '%s'";
+        query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'";
     } else if (!strcmp(stage, "diff")) {
-        query = "UPDATE diffSkyfile SET magicked = 1 where diff_id = %" PRId64 " AND skycell_id = '%s'";
+        query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'";
     } else {
         psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage);
@@ -517,5 +517,5 @@
         return false;
     }
-    if (!p_psDBRunQueryF(config->dbh, query, stage_id, component)) {
+    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -567,11 +567,11 @@
     // chose the appropriate query based on the stage
     if (!strcmp(stage, "raw")) {
-        query = "UPDATE rawExp SET magicked = 1 where exp_id = %" PRId64;
+        query = "UPDATE rawExp SET magicked = %" PRId64 " where exp_id = %" PRId64;
     } else if (!strcmp(stage, "chip")) {
-        query = "UPDATE chipRun SET magicked = 1 where chip_id = %" PRId64;
+        query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64;
     } else if (!strcmp(stage, "warp")) {
-        query = "UPDATE warpRun SET magicked = 1 where warp_id = %" PRId64;
+        query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64;
     } else if (!strcmp(stage, "diff")) {
-        query = "UPDATE diffRun SET magicked = 1 where diff_id = %" PRId64;
+        query = "UPDATE diffRun SET magicked = %" PRId64 " where diff_id = %" PRId64;
     } else {
         psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage);
@@ -579,5 +579,5 @@
         return false;
     }
-    if (!p_psDBRunQueryF(config->dbh, query, stage_id)) {
+    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -629,5 +629,5 @@
     }
 
-    if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault)) {
+    if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault, "full")) {
             // rollback
         if (!psDBRollback(config->dbh)) {
Index: trunk/ippTools/src/warptool.c
===================================================================
--- trunk/ippTools/src/warptool.c	(revision 24551)
+++ trunk/ippTools/src/warptool.c	(revision 24552)
@@ -163,5 +163,5 @@
             end_stage,
             registered,
-            false       // magicked
+            0       // magicked
     );
     if (!warpRun) {
@@ -980,5 +980,5 @@
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
-    PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false);
+    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
 
     // default values
@@ -1084,5 +1084,5 @@
             return false;
         }
-        psS32 magicked = psMetadataLookupS64(&status, row, "magicked");
+        psS64 magicked = psMetadataLookupS64(&status, row, "magicked");
         if (!status) {
             psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked");