Changeset 26567 for trunk/ippTools

Timestamp:

Jan 12, 2010, 2:07:51 PM (17 years ago)

Author:

bills

Message:

Make update processing compatible with magic. Fix some bugs in update for the warp stage
Make script changes to implement update for diff stage.
Change value of 'magicked' in the various runs and files to contain the value
the magic_id of the magicRun that corresponds to the streaks file used for desreaking.

Location:

trunk/ippTools

Files:

• share/camtool_find_pendingexp.sql (modified) (1 diff)
• share/camtool_queue_chip_id.sql (modified) (1 diff)
• share/chiptool_change_exp_state.sql (modified) (1 diff)
• share/chiptool_completely_processed_exp.sql (modified) (2 diffs)
• share/difftool_change_run_state.sql (modified) (1 diff)
• share/difftool_change_skyfile_data_state.sql (modified) (1 diff)
• share/difftool_todiffskyfile.sql (modified) (1 diff)
• share/disttool_pending_camera.sql (modified) (1 diff)
• share/disttool_pending_chip.sql (modified) (1 diff)
• share/disttool_pending_diff.sql (modified) (1 diff)
• share/disttool_pending_warp.sql (modified) (1 diff)
• share/warptool_change_run_state.sql (modified) (2 diffs)
• share/warptool_change_skyfile_data_state.sql (modified) (1 diff)
• share/warptool_finish_run.sql (modified) (1 diff)
• share/warptool_finished_run_select.sql (modified) (1 diff)
• share/warptool_towarped.sql (modified) (1 diff)
• src/camtool.c (modified) (4 diffs)
• src/chiptool.c (modified) (4 diffs)
• src/chiptoolConfig.c (modified) (1 diff)
• src/difftool.c (modified) (8 diffs)
• src/difftool.h (modified) (1 diff)
• src/difftoolConfig.c (modified) (3 diffs)
• src/flatcorr.c (modified) (1 diff)
• src/magicdstool.c (modified) (15 diffs)
• src/magicdstoolConfig.c (modified) (1 diff)
• src/pxcam.c (modified) (3 diffs)
• src/pxcam.h (modified) (1 diff)
• src/pxchip.c (modified) (4 diffs)
• src/pxchip.h (modified) (1 diff)
• src/warptool.c (modified) (2 diffs)
• src/warptoolConfig.c (modified) (2 diffs)

trunk/ippTools/share/camtool_find_pendingexp.sql

@@ -9 +9 @@
     rawExp.camera,
     rawExp.telescope,
-    rawExp.filelevel
+    rawExp.filelevel,
+    chipRun.magicked AS chip_magicked
 FROM camRun
 JOIN chipRun

trunk/ippTools/share/camtool_queue_chip_id.sql

@@ -16 +16 @@
         '%s',           -- tess_id
         '%s',           -- end_stage
-        0,              -- magicked
-        '%s'           -- note
+        %lld,           -- magicked
+        '%s'            -- note
     FROM chipRun
     WHERE

trunk/ippTools/share/chiptool_change_exp_state.sql

@@ -4 +4 @@
 -- the chipProcessedImfile sub query
 UPDATE chipRun
-    SET state = '%s'
+JOIN rawExp using(exp_id)
+    SET chipRun.state = '%s'
+    -- set magicked hook %s
     WHERE
     chipRun.chip_id = %lld

trunk/ippTools/share/chiptool_completely_processed_exp.sql

@@ -14 +14 @@
     tess_id,
     end_stage,
-    all_files_magicked as magicked,
+    imfile_magicked as magicked,
     note
 FROM
@@ -21 +21 @@
         rawImfile.class_id as rawimfile_class_id,
         chipProcessedImfile.class_id,
-        SUM(!chipProcessedImfile.magicked) = 0 as all_files_magicked
+        -- XXX using chipProcessedImfile assumes that all imfile's have
+        -- the same magicked value if that isn't right then more than one 
+        -- row will be returned. In practice this is the case but bugs could
+        -- cause it to not be true.
+        -- We could use rawExp.magicked but that would make it possible for
+        -- the chipRun to have a different magicked value than the imfiles
+        chipProcessedImfile.magicked AS imfile_magicked
     FROM chipRun
     JOIN rawImfile

trunk/ippTools/share/difftool_change_run_state.sql

@@ -5 +5 @@
 UPDATE diffRun
     SET state = '%s'
+    -- set magicked hook %s
     WHERE
     diffRun.diff_id = %lld

trunk/ippTools/share/difftool_change_skyfile_data_state.sql

@@ -5 +5 @@
     SET 
     data_state = '%s'
+    -- magicked hook %s
 WHERE
     diff_id = %lld

trunk/ippTools/share/difftool_todiffskyfile.sql

@@ -65 +65 @@
     OR (diffRun.state = 'update'
     AND diffSkyfile.fault = 0
-    AND diffSkyfile.quality = 0)
+    AND diffSkyfile.data_state = 'cleaned')
     )
 -- Ensure input warps are available

trunk/ippTools/share/disttool_pending_camera.sql

@@ -30 +30 @@
     AND distRun.stage = 'camera'
     AND distComponent.dist_id IS NULL
-    AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic)
+    AND (((chipRun.magicked > 0) AND (camRun.magicked > 0)) OR distRun.no_magic)
     AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))

trunk/ippTools/share/disttool_pending_chip.sql

@@ -28 +28 @@
     AND distRun.stage = 'chip'
     AND distComponent.dist_id IS NULL
-    AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
+    AND ((chipRun.magicked > 0) OR distRun.no_magic)
     AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned'))

trunk/ippTools/share/disttool_pending_diff.sql

@@ -36 +36 @@
     AND distRun.stage = 'diff'
     AND distComponent.dist_id IS NULL
-    AND (distRun.clean OR diffRun.magicked OR distRun.no_magic)
+    AND ((diffRun.magicked > 0) OR distRun.no_magic)
     AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))

trunk/ippTools/share/disttool_pending_warp.sql

@@ -31 +31 @@
     AND distRun.stage = 'warp'
     AND distComponent.dist_id IS NULL
-    AND (distRun.clean OR warpRun.magicked OR distRun.no_magic)
+    AND ((warpRun.magicked > 0) OR distRun.no_magic)
     AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned'))

trunk/ippTools/share/warptool_change_run_state.sql

@@ -4 +4 @@
 -- the chipProcessedImfile sub query
 UPDATE warpRun
-    SET state = '%s'
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+    SET warpRun.state = '%s'
+    -- set magicked  hook %s
     WHERE
     warpRun.warp_id = %lld
@@ -12 +16 @@
         WHERE
             warpSkyfile.warp_id = warpRun.warp_id
-            AND data_state != '%s'
+            AND warpSkyfile.data_state != '%s'
         ) = 0

trunk/ippTools/share/warptool_change_skyfile_data_state.sql

@@ -1 +1 @@
 -- handle changes in warpSkyfile.data_state.
--- Used for the modes tocleanedskyfile and topurgedskyfile
--- args are new data_state, warp_id, skycell_id and current expected state for warpRun
+-- Used for the modes tofullskyfile, tocleanedskyfile and topurgedskyfile
+-- args are new data_state, string for magic hook, warp_id, skycell_id and current expected state for warpRun
 UPDATE warpSkyfile
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
     SET 
     data_state = '%s'
+    -- set magicked hook %s
 WHERE
     warp_id = %lld

trunk/ippTools/share/warptool_finish_run.sql

@@ -1 +1 @@
 UPDATE warpRun
     SET state = 'full',
-    magicked = %d
+    magicked = %lld
 WHERE warp_id = %lld
 

trunk/ippTools/share/warptool_finished_run_select.sql

@@ -8 +8 @@
        warpSkyCellMap.warp_id as foo,
        warpSkyfile.warp_id as bar,
-       SUM(!warpSkyfile.magicked) = 0 as magicked
+       warpSkyfile.magicked as magicked
    FROM warpRun
    JOIN warpSkyCellMap

trunk/ippTools/share/warptool_towarped.sql

@@ -40 +40 @@
         AND warpSkyfile.tess_id IS NULL)
     OR (warpRun.state = 'update'
+        AND chipRun.magicked >= 0
+        AND warpSkyfile.fault = 0
         AND warpSkyfile.data_state = 'cleaned')
     )

trunk/ippTools/src/camtool.c

@@ -222 +222 @@
                     tess_id     ? tess_id   : row->tess_id,
                     end_stage   ? end_stage : row->end_stage,
+                    row->magicked,
                     note
         )) {
@@ -480 +481 @@
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
 
-    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
+//    Get this from the chipRun
+//    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
 
     // generate restrictions
@@ -523 +525 @@
         return false;
     }
+
+    bool status;
+    psS64 chip_magicked = psMetadataLookupS64(&status, output->data[0], "chip_magicked");
 
     camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]);
@@ -615 +620 @@
 
     // since there is only one exp per 'new' set camRun.state = 'full'
-    if (!pxcamRunSetState(config, row->cam_id, "full", magicked)) {
+    // propagate magicked state from chipRun
+    if (!pxcamRunSetState(config, row->cam_id, "full", chip_magicked)) {
         psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id);
         psFree(row);

trunk/ippTools/src/chiptool.c

@@ -720 +720 @@
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
     PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL);
+    if (psListLength(where->list) == 0) {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        return false;
+    }
         
     if (!state) {
@@ -1254 +1258 @@
                     chipRun->tess_id,
                     chipRun->end_stage,
+                    chipRun->magicked,
                     NULL    // note does not propagate
         )) {
@@ -1295 +1300 @@
     }
 
-    psString set_magic = "";
+    char *set_magicked_imfile = "";
+    char *set_magicked_run = "";
     if (!strcmp(data_state, "full")) {
-        // copy the magicked state from the input to the output when transitioning to full state
-        set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked";
-    }
-
-    // note only updates if chipRun.state = run_state
-    // XXX note that we have removed this constraint for now
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) {
+        // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will
+        // block warp processing until destreaking has been done
+        // otherwise copy magicked from the rawImfile
+        // Same thing for chipRun/rawExp
+        set_magicked_imfile = "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0 AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)";
+        set_magicked_run = "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), chipRun.magicked, rawExp.magicked)";
+    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
+        // if magicked is non-zero set it to -1
+        set_magicked_imfile = "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)";
+        set_magicked_run = "\n, chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)";
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) {
         psFree(query);
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1320 +1332 @@
 
     query = pxDataGet("chiptool_change_exp_state.sql");
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) {
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) {
         psFree(query);
         // rollback

trunk/ippTools/src/chiptoolConfig.c

@@ -154 +154 @@
     psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
-    psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", false);
+    psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", 0);
 
     // -processedimfile

trunk/ippTools/src/difftool.c

@@ -55 +55 @@
 static bool topurgedskyfileMode(pxConfig *config);
 static bool toscrubbedskyfileMode(pxConfig *config);
+static bool tofullskyfileMode(pxConfig *config);
 
 
@@ -94 +95 @@
         MODECASE(DIFFTOOL_MODE_EXPORTRUN,             exportrunMode);
         MODECASE(DIFFTOOL_MODE_IMPORTRUN,             importrunMode);
-        MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
-        MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE,    topurgedskyfileMode);
-        MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,  toscrubbedskyfileMode);
+        MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE,      tocleanedskyfileMode);
+        MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE,       topurgedskyfileMode);
+        MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,     toscrubbedskyfileMode);
+        MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE,         tofullskyfileMode);
 
         default:
@@ -2036 +2038 @@
   PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
 
+
+  psS64 magicked = 0;
+  if (!strcmp(data_state, "full")) {
+      PXOPT_LOOKUP_S64(set_magicked, config->args, "-magicked", 0, false);
+      magicked = set_magicked;
+  } 
+
   psString query = pxDataGet("difftool_change_skyfile_data_state.sql");
 
@@ -2043 +2052 @@
   }
 
+  psString set_magicked_skyfile = psStringCopy("");
+  psString set_magicked_run = psStringCopy("");
+  if (magicked != 0 && !strcmp(data_state, "full")) {
+    psStringAppend(&set_magicked_skyfile,  "\n , diffSkyfile.magicked = %" PRId64, magicked);
+    psStringAppend(&set_magicked_run,      "\n , diffRun.magicked = %" PRId64, magicked);
+
+  } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
+    psStringAppend(&set_magicked_skyfile, "\n, diffSkyfile.magicked = IF(diffSkyfile.magicked = 0, 0, -1)");
+    psStringAppend(&set_magicked_run, "\n, diffRun.magicked = IF(diffRun.magicked = 0, 0, -1)");
+  }
+
   // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c
 
-  if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, skycell_id)) {
+  if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, diff_id, skycell_id)) {
     psError(PS_ERR_UNKNOWN, false, "database error");
     // rollback
@@ -2052 +2072 @@
     }
     psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(set_magicked_skyfile);
     return(false);
   }
+  psFree(set_magicked_skyfile);
   psFree(query);
 
   query = pxDataGet("difftool_change_run_state.sql");
-  if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, data_state)) {
+  if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, diff_id, data_state)) {
     // rollback
     if (!psDBRollback(config->dbh)) {
@@ -2063 +2085 @@
     }
     psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(set_magicked_run);
     return(false);
   }
@@ -2068 +2091 @@
   if (!psDBCommit(config->dbh)) {
     psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(set_magicked_run);
     return(false);
   }
+  psFree(set_magicked_run);
 
   return(true);
@@ -2082 +2107 @@
 static bool toscrubbedskyfileMode(pxConfig *config) {
   return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed");
+}
+static bool tofullskyfileMode(pxConfig *config) {
+  return change_skyfile_data_state(config, "full", "update");
 }
 

trunk/ippTools/src/difftool.h

@@ -45 +45 @@
     DIFFTOOL_MODE_TOCLEANEDSKYFILE,
     DIFFTOOL_MODE_TOPURGEDSKYFILE,
-    DIFFTOOL_MODE_TOSCRUBBEDSKYFILE
+    DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,
+    DIFFTOOL_MODE_TOFULLSKYFILE
     
 } difftoolMode;

trunk/ippTools/src/difftoolConfig.c

@@ -122 +122 @@
     psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0,   "define hostname", 0);
     psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac",  0, "define %% of good pixels", NAN);
-    psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", false);
+    psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", 0);
 
     // -advance
@@ -289 +289 @@
     psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
 
+    // -tofullskyfile
+    psMetadata *tofullskyfileArgs = psMetadataAlloc();
+    psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0);
+    psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
+    psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", 0);
+
     // -exportrun
     psMetadata *exportrunArgs = psMetadataAlloc();
@@ -329 +335 @@
     PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs);
     PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs);
+    PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

trunk/ippTools/src/flatcorr.c

@@ -535 +535 @@
                 row->tess_id,
                 "camera",
+                row->magicked,
                 NULL // note does not propragate
                 )) {

trunk/ippTools/src/magicdstool.c

@@ -44 +44 @@
 static bool tocleanupMode(pxConfig *config);
 
-static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state);
+static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state);
 
 # define MODECASE(caseName, func) \
@@ -326 +326 @@
     PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
 
+    // optional
+    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
+    psString setString = NULL;
+    if (set_label) {
+        psStringAppend(&setString, ", label = '%s'", set_label);
+    } 
+
     PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false);
     if (magic_ds_id) {
 
-        return setmagicDSRunState(config, magic_ds_id, NULL, state);
+        return setmagicDSRunState(config, magic_ds_id, setString, NULL, state);
 
     } else if (!strcmp(state, "full")) {
@@ -336 +343 @@
     }
     // we can transition by query as well
+
 
     psMetadata *where = psMetadataAlloc();
@@ -342 +350 @@
     PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
 
-
     if (psListLength(where->list) < 2) {
         psError(PS_ERR_UNKNOWN, true, "at least 2 search arguments are required");
@@ -353 +360 @@
         psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0);
     }
-    bool result = setmagicDSRunState(config, magic_ds_id, where, state);
+    bool result = setmagicDSRunState(config, magic_ds_id, setString, where, state);
     psFree(where);
 
@@ -445 +452 @@
 {
     // first query the magicDSRun to find the stage and the stage_id
-    psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
+    psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64;
 
     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
@@ -472 +479 @@
     psString stage= psMetadataLookupStr(NULL, row, "stage");
     psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
+    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
 
     ippStage stageNum = ippStringToStage(stage);
@@ -499 +507 @@
     }
 
-    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) {
+    if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -518 +526 @@
 {
     // first query the magicDSRun to find the stage and the stage_id
-    psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
+    psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64;
 
     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
@@ -545 +553 @@
     psString stage = psMetadataLookupStr(NULL, row, "stage");
     psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
+    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
 
     ippStage stageNum = ippStringToStage(stage);
@@ -570 +579 @@
         return false;
     }
-    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) {
+    if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -710 +719 @@
 
         // set magicDSRun.state to 'full'
-        if (!setmagicDSRunState(config, magic_ds_id, NULL, "full")) {
+        if (!setmagicDSRunState(config, magic_ds_id, NULL, NULL, "full")) {
             psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64,
                 magic_ds_id);
@@ -927 +936 @@
 }
 
-static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psMetadata *where, const char *state)
+static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psString extraSetStr, psMetadata *where, const char *state)
 {
     PS_ASSERT_PTR_NON_NULL(state, false);
@@ -937 +946 @@
     }
 
-    psString query = psStringCopy("UPDATE magicDSRun SET state = '%s'");
+    psString query = NULL;
+    psStringAppend(&query, "UPDATE magicDSRun SET state = '%s' %s\n", state, extraSetStr ? extraSetStr : "");
     if (magic_ds_id) {
         psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id);
@@ -949 +959 @@
     }
 
-    if (!p_psDBRunQueryF(config->dbh, query, state)) {
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false,
                 "failed to change state for magic_id %" PRId64, magic_ds_id);

trunk/ippTools/src/magicdstoolConfig.c

@@ -74 +74 @@
     psMetadata *updaterunArgs = psMetadataAlloc();
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL);
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage",     0, "define stage", NULL);

trunk/ippTools/src/pxcam.c

@@ -152 +152 @@
                     char *tess_id,
                     char *end_stage,
+                    psS64 magicked,
                     char *note)
 {
@@ -168 +169 @@
 
     // queue the exp
-    // XXX chip_id is being cast here work around psS64 have a different type
+    // XXX chip_id and magicked are being cast here work around psS64 have a different type
     // different on 32/64
     if (!p_psDBRunQueryF(config->dbh, query,
@@ -182 +183 @@
                 tess_id  ? tess_id  : "NULL",
                 end_stage ? end_stage : "NULL",
+                (long long) magicked,
                 note     ? note     : "NULL",
                 (long long)chip_id

trunk/ippTools/src/pxcam.h

@@ -40 +40 @@
                         char *tess_id,
                         char *end_stage,
+                        psS64 magicked,
                         char *note);
 

trunk/ippTools/src/pxchip.c

@@ -129 +129 @@
 }
 
-bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked)
+bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -140 +140 @@
     }
 
-    char *query = "UPDATE chipRun SET state = '%s', magicked = %d WHERE chip_id = %" PRId64;
+    char *query = "UPDATE chipRun SET state = '%s', magicked = %" PRId64 " WHERE chip_id = %" PRId64;
     if (!p_psDBRunQueryF(config->dbh, query, state, magicked, chip_id)) {
         psError(PS_ERR_UNKNOWN, false,
@@ -174 +174 @@
 
     psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'");
+
+    if (!strcmp(state, "cleaned") || !strcmp(state, "purged")) {
+        // if magicked is non-zero set it to -1
+        psStringAppend(&query, ", chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)");
+    }
 
     if (where && psListLength(where->list) > 0) {
@@ -219 +224 @@
         psStringAppend(&query, " %s", whereClause);
         psFree(whereClause);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        return false;
     }
 

trunk/ippTools/src/pxchip.h

@@ -27 +27 @@
 
 
-bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked);
+bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked);
 bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state);
 

trunk/ippTools/src/warptool.c

@@ -1646 +1646 @@
     }
 
-    // note only updates if warpRun.state = run_state
-    // XXX note that we have removed this constraint for now
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) {
+    char *set_magicked_skyfile = "";
+    char *set_magicked_run = "";
+    if (!strcmp(data_state, "full")) {
+        set_magicked_skyfile = "\n , warpSkyfile.magicked = IF((warpSkyfile.magicked < 0 AND chipRun.magicked = 0), warpSkyfile.magicked, chipRun.magicked)";
+        set_magicked_run = "\n,  warpRun.magicked = IF((warpRun.magicked < 0 AND chipRun.magicked = 0), warpRun.magicked, chipRun.magicked)";
+
+    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
+        set_magicked_skyfile = "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1)";
+        set_magicked_run = "\n, warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)";
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         // rollback
@@ -1660 +1669 @@
 
     query = pxDataGet("warptool_change_run_state.sql");
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, data_state)) {
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) {
         // rollback
         if (!psDBRollback(config->dbh)) {

trunk/ippTools/src/warptoolConfig.c

@@ -212 +212 @@
     psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
     psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
-    psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", false);
+    psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", 0);
     psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
@@ -310 +310 @@
     psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
 
-    // -toupdateskyfile
+    // -updateskyfile
     psMetadata *updateskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,    "warptool ID to update", 0);