Index: trunk/ippScripts/scripts/chip_imfile.pl
===================================================================
--- trunk/ippScripts/scripts/chip_imfile.pl	(revision 26566)
+++ trunk/ippScripts/scripts/chip_imfile.pl	(revision 26567)
@@ -53,5 +53,5 @@
     'reduction=s'       => \$reduction, # Reduction class
     'run-state=s'       => \$run_state, # current state of the run (new, update)
-    'magicked'          => \$magicked,  # magicked state of input file
+    'magicked=s'        => \$magicked,  # magicked state of input file
     'deburned=s'        => \$deburned,  # does deburned image exist?
     'threads=s'         => \$threads,   # Number of threads to use for ppImage
@@ -411,5 +411,5 @@
     $command .= " -uri $outputImage";
     $command .= " -path_base $outroot";
-    $command .= " -magicked" if $magicked;
+    $command .= " -magicked $magicked" if $magicked;
     $command .= " -hostname $host" if defined $host;
     $command .= " -dbname $dbname" if defined $dbname;
Index: trunk/ippScripts/scripts/diff_skycell.pl
===================================================================
--- trunk/ippScripts/scripts/diff_skycell.pl	(revision 26566)
+++ trunk/ippScripts/scripts/diff_skycell.pl	(revision 26567)
@@ -35,5 +35,5 @@
 }
 
-my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $verbose, $no_update, $no_op, $redirect);
+my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect);
 my ($skycell_id, $diff_skyfile_id);
 GetOptions(
@@ -43,4 +43,5 @@
     'dbname|d=s'        => \$dbname, # Database name
     'threads=s'         => \$threads,   # Number of threads to use
+    'run-state=s'       => \$run_state,   # state for run: 'new' or 'update'
     'outroot=s'         => \$outroot, # Output root name
     'inverse'           => \$inverse, # Make inverse subtraction?
@@ -54,9 +55,10 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-    -msg => "Required options: --diff_id --skycell_id --outroot",
+    -msg => "Required options: --diff_id --skycell_id --outroot --run-state",
     -exitval => 3,
           ) unless defined $diff_id
     and defined $skycell_id
     and defined $diff_skyfile_id
+    and defined $run_state
     and defined $outroot;
 
@@ -66,4 +68,5 @@
 # my $logDest = $ipprc->filename("LOG.EXP", $outroot);
 my $logDest = "$outroot.log";
+$logDest .= ".update" if $run_state eq "update";
 $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 
@@ -155,7 +158,5 @@
 # The output file will be considered magicked if the input has been magicked and the
 # template is either a stack or a warp that has been magicked.
-# note that difftool -inputskyfile outputs the magicked boolean as an int not T or F
-# because the output is constructed from a union of two selects
-my $magicked = $inputMagic && $templateMagic;
+my $magicked = $inputMagic && $templateMagic ? $inputMagic : 0;
 
 # Recipes to use based on reduction class
@@ -212,4 +213,9 @@
 my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot);
 my $traceDest = $ipprc->filename("TRACE.EXP", $outroot);
+
+if ($run_state eq 'update') {
+    $traceDest .= '.update';
+    $outputStats .= '.update';
+}
 
 my ($inverseName, $inverseMask, $inverseVariance, $inverseSources);
@@ -236,4 +242,10 @@
     $command .= " -stats $outputStats";
     $command .= " -threads $threads" if defined $threads;
+    my $configurationReal = $ipprc->file_resolve($configuration);
+    if ($run_state eq "new") {
+        $command .= " -dumpconfig $configurationReal";
+    } else {
+        $command .= " -ipprc $configurationReal";
+    }
     $command .= " -recipe PPSUB $recipe_ppSub";
     $command .= " -recipe PSPHOT $recipe_psphot";
@@ -245,5 +257,4 @@
     $command .= " -inverse" if $inverse;
     $command .= " -tracedest $traceDest -log $logDest";
-    $command .= " -dumpconfig $configuration";
     $command .= " -dbname $dbname" if defined $dbname;
     $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id;
@@ -295,9 +306,14 @@
     # Add the subtraction result
     {
-        my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -path_base $outroot";
-        $command .= " $cmdflags";
-        $command .= " -magicked" if $magicked;
-        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
-        $command .= " -hostname $host" if defined $host;
+        my $command = "$difftool -diff_id $diff_id -skycell_id $skycell_id";
+        $command .= " -magicked $magicked" if $magicked;
+        if ($run_state eq 'new') {
+            $command .= " -adddiffskyfile -path_base $outroot";
+            $command .= " $cmdflags";
+            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+            $command .= " -hostname $host" if defined $host;
+        } else {
+            $command .= " -tofullskyfile";
+        }
         $command .= " -dbname $dbname" if defined $dbname;
 
@@ -306,5 +322,8 @@
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform difftool -adddiffskyfile: $error_code", $diff_id, $skycell_id, $error_code);
+            my $err_message = $run_state eq "update" ? 
+                "Unable to perform difftool -adddiffskyfile" :
+                "Unable to perform difftool -tofullskyfile";
+            &my_die("$err_message: $error_code", $diff_id, $skycell_id, $error_code);
         }
     }
@@ -323,9 +342,14 @@
     warn($msg);
     if (defined $diff_id and defined $skycell_id and not $no_update) {
-        my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
-        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
-        $command .= " -hostname $host" if defined $host;
-        $command .= " -path_base $outroot" if defined $outroot;
-        $command .= " -dbname $dbname" if defined $dbname;
+        my $command = "$difftool -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
+        if ($run_state eq 'new') {
+            $command .= " -adddiffskyfile";
+            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+            $command .= " -hostname $host" if defined $host;
+            $command .= " -path_base $outroot" if defined $outroot;
+            $command .= " -dbname $dbname" if defined $dbname;
+        } else {
+            $command .= " -updatediffskyfile";
+        }
         run(command => $command, verbose => $verbose);
     }
Index: trunk/ippScripts/scripts/warp_skycell.pl
===================================================================
--- trunk/ippScripts/scripts/warp_skycell.pl	(revision 26566)
+++ trunk/ippScripts/scripts/warp_skycell.pl	(revision 26567)
@@ -48,5 +48,5 @@
     'threads=s'           => \$threads,   # Number of threads to use for pswarp
     'run-state=s'         => \$run_state,  # 'new' or 'update'
-    'magicked'            => \$magicked,  # input run has been magicked already?
+    'magicked=s'          => \$magicked,  # input run has been magicked already?
     'verbose'             => \$verbose,   # Print to stdout
     'no-update'           => \$no_update, # Don't update the database?
@@ -276,5 +276,5 @@
             $command .= " -tess_id $tess_dir";
             $command .= " -path_base $outroot"; # needed for logfile lookups
-            $command .= " -magicked" if $magicked;
+            $command .= " -magicked $magicked" if $magicked;
 
             $command .= " -uri $outputImage" if !$quality;
Index: trunk/ippTasks/chip.pro
===================================================================
--- trunk/ippTasks/chip.pro	(revision 26566)
+++ trunk/ippTasks/chip.pro	(revision 26567)
@@ -138,6 +138,6 @@
     book getword chipPendingImfile $pageName state -var RUN_STATE
 
-    if ("$RAW_MAGICKED" == "T")
-        $MAGICKED_ARG = "--magicked"
+    if ($RAW_MAGICKED > 0)
+        $MAGICKED_ARG = "--magicked $RAW_MAGICKED"
     else 
         $MAGICKED_ARG = ""
Index: trunk/ippTasks/diff.pro
===================================================================
--- trunk/ippTasks/diff.pro	(revision 26566)
+++ trunk/ippTasks/diff.pro	(revision 26567)
@@ -155,4 +155,5 @@
     book getword diffSkyfile $pageName bothways -var BOTHWAYS
     book getword diffSkyfile $pageName workdir -var WORKDIR_TEMPLATE
+    book getword diffSkyfile $pageName state -var RUN_STATE
     book getword diffSkyfile $pageName dbname -var DBNAME
 
@@ -171,5 +172,5 @@
     stderr $LOGDIR/diff.skycell.log
 
-    $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output
+    $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output --run-state $RUN_STATE
     if ("$BOTHWAYS" == "T")
        $run = $run --inverse
Index: trunk/ippTasks/dist.cleanup.pro
===================================================================
--- trunk/ippTasks/dist.cleanup.pro	(revision 26566)
+++ trunk/ippTasks/dist.cleanup.pro	(revision 26567)
@@ -48,9 +48,9 @@
 
   stdout NULL
-  stderr $LOGDIR/dist.cleanup.log
+  stderr $LOGDIR/dist.cleanup.load.log
 
   task.exec
     if ($LABEL:n == 0) break
-    $run = disttool -pendingcleanup
+    $run = disttool -pendingcleanup -all
     if ($DB:n == 0)
       option DEFAULT
@@ -98,5 +98,5 @@
   periods      -poll $RUNPOLL
   periods      -exec $RUNEXEC
-  periods      -timeout 60
+  periods      -timeout 600
   active       false
 
Index: trunk/ippTasks/warp.pro
===================================================================
--- trunk/ippTasks/warp.pro	(revision 26566)
+++ trunk/ippTasks/warp.pro	(revision 26567)
@@ -295,6 +295,6 @@
     book getword warpPendingSkyCell $pageName state -var RUN_STATE
     book getword warpPendingSkyCell $pageName magicked -var CHIP_MAGICKED
-    if ("$CHIP_MAGICKED" == "T")
-        $MAGICKED_ARG = "--magicked"
+    if ($CHIP_MAGICKED > 0)
+        $MAGICKED_ARG = "--magicked $CHIP_MAGICKED"
     else
         $MAGICKED_ARG = ""
Index: trunk/ippTools/share/camtool_find_pendingexp.sql
===================================================================
--- trunk/ippTools/share/camtool_find_pendingexp.sql	(revision 26566)
+++ trunk/ippTools/share/camtool_find_pendingexp.sql	(revision 26567)
@@ -9,5 +9,6 @@
     rawExp.camera,
     rawExp.telescope,
-    rawExp.filelevel
+    rawExp.filelevel,
+    chipRun.magicked AS chip_magicked
 FROM camRun
 JOIN chipRun
Index: trunk/ippTools/share/camtool_queue_chip_id.sql
===================================================================
--- trunk/ippTools/share/camtool_queue_chip_id.sql	(revision 26566)
+++ trunk/ippTools/share/camtool_queue_chip_id.sql	(revision 26567)
@@ -16,6 +16,6 @@
         '%s',           -- tess_id
         '%s',           -- end_stage
-        0,              -- magicked
-        '%s'           -- note
+        %lld,           -- magicked
+        '%s'            -- note
     FROM chipRun
     WHERE
Index: trunk/ippTools/share/chiptool_change_exp_state.sql
===================================================================
--- trunk/ippTools/share/chiptool_change_exp_state.sql	(revision 26566)
+++ trunk/ippTools/share/chiptool_change_exp_state.sql	(revision 26567)
@@ -4,5 +4,7 @@
 -- the chipProcessedImfile sub query
 UPDATE chipRun
-    SET state = '%s'
+JOIN rawExp using(exp_id)
+    SET chipRun.state = '%s'
+    -- set magicked hook %s
     WHERE
     chipRun.chip_id = %lld
Index: trunk/ippTools/share/chiptool_completely_processed_exp.sql
===================================================================
--- trunk/ippTools/share/chiptool_completely_processed_exp.sql	(revision 26566)
+++ trunk/ippTools/share/chiptool_completely_processed_exp.sql	(revision 26567)
@@ -14,5 +14,5 @@
     tess_id,
     end_stage,
-    all_files_magicked as magicked,
+    imfile_magicked as magicked,
     note
 FROM
@@ -21,5 +21,11 @@
         rawImfile.class_id as rawimfile_class_id,
         chipProcessedImfile.class_id,
-        SUM(!chipProcessedImfile.magicked) = 0 as all_files_magicked
+        -- XXX using chipProcessedImfile assumes that all imfile's have
+        -- the same magicked value if that isn't right then more than one 
+        -- row will be returned. In practice this is the case but bugs could
+        -- cause it to not be true.
+        -- We could use rawExp.magicked but that would make it possible for
+        -- the chipRun to have a different magicked value than the imfiles
+        chipProcessedImfile.magicked AS imfile_magicked
     FROM chipRun
     JOIN rawImfile
Index: trunk/ippTools/share/difftool_change_run_state.sql
===================================================================
--- trunk/ippTools/share/difftool_change_run_state.sql	(revision 26566)
+++ trunk/ippTools/share/difftool_change_run_state.sql	(revision 26567)
@@ -5,4 +5,5 @@
 UPDATE diffRun
     SET state = '%s'
+    -- set magicked hook %s
     WHERE
     diffRun.diff_id = %lld
Index: trunk/ippTools/share/difftool_change_skyfile_data_state.sql
===================================================================
--- trunk/ippTools/share/difftool_change_skyfile_data_state.sql	(revision 26566)
+++ trunk/ippTools/share/difftool_change_skyfile_data_state.sql	(revision 26567)
@@ -5,4 +5,5 @@
     SET 
     data_state = '%s'
+    -- magicked hook %s
 WHERE
     diff_id = %lld
Index: trunk/ippTools/share/difftool_todiffskyfile.sql
===================================================================
--- trunk/ippTools/share/difftool_todiffskyfile.sql	(revision 26566)
+++ trunk/ippTools/share/difftool_todiffskyfile.sql	(revision 26567)
@@ -65,5 +65,5 @@
     OR (diffRun.state = 'update'
     AND diffSkyfile.fault = 0
-    AND diffSkyfile.quality = 0)
+    AND diffSkyfile.data_state = 'cleaned')
     )
 -- Ensure input warps are available
Index: trunk/ippTools/share/disttool_pending_camera.sql
===================================================================
--- trunk/ippTools/share/disttool_pending_camera.sql	(revision 26566)
+++ trunk/ippTools/share/disttool_pending_camera.sql	(revision 26567)
@@ -30,4 +30,4 @@
     AND distRun.stage = 'camera'
     AND distComponent.dist_id IS NULL
-    AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic)
+    AND (((chipRun.magicked > 0) AND (camRun.magicked > 0)) OR distRun.no_magic)
     AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))
Index: trunk/ippTools/share/disttool_pending_chip.sql
===================================================================
--- trunk/ippTools/share/disttool_pending_chip.sql	(revision 26566)
+++ trunk/ippTools/share/disttool_pending_chip.sql	(revision 26567)
@@ -28,4 +28,4 @@
     AND distRun.stage = 'chip'
     AND distComponent.dist_id IS NULL
-    AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
+    AND ((chipRun.magicked > 0) OR distRun.no_magic)
     AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned'))
Index: trunk/ippTools/share/disttool_pending_diff.sql
===================================================================
--- trunk/ippTools/share/disttool_pending_diff.sql	(revision 26566)
+++ trunk/ippTools/share/disttool_pending_diff.sql	(revision 26567)
@@ -36,4 +36,4 @@
     AND distRun.stage = 'diff'
     AND distComponent.dist_id IS NULL
-    AND (distRun.clean OR diffRun.magicked OR distRun.no_magic)
+    AND ((diffRun.magicked > 0) OR distRun.no_magic)
     AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))
Index: trunk/ippTools/share/disttool_pending_warp.sql
===================================================================
--- trunk/ippTools/share/disttool_pending_warp.sql	(revision 26566)
+++ trunk/ippTools/share/disttool_pending_warp.sql	(revision 26567)
@@ -31,4 +31,4 @@
     AND distRun.stage = 'warp'
     AND distComponent.dist_id IS NULL
-    AND (distRun.clean OR warpRun.magicked OR distRun.no_magic)
+    AND ((warpRun.magicked > 0) OR distRun.no_magic)
     AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned'))
Index: trunk/ippTools/share/warptool_change_run_state.sql
===================================================================
--- trunk/ippTools/share/warptool_change_run_state.sql	(revision 26566)
+++ trunk/ippTools/share/warptool_change_run_state.sql	(revision 26567)
@@ -4,5 +4,9 @@
 -- the chipProcessedImfile sub query
 UPDATE warpRun
-    SET state = '%s'
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+    SET warpRun.state = '%s'
+    -- set magicked  hook %s
     WHERE
     warpRun.warp_id = %lld
@@ -12,4 +16,4 @@
         WHERE
             warpSkyfile.warp_id = warpRun.warp_id
-            AND data_state != '%s'
+            AND warpSkyfile.data_state != '%s'
         ) = 0
Index: trunk/ippTools/share/warptool_change_skyfile_data_state.sql
===================================================================
--- trunk/ippTools/share/warptool_change_skyfile_data_state.sql	(revision 26566)
+++ trunk/ippTools/share/warptool_change_skyfile_data_state.sql	(revision 26567)
@@ -1,8 +1,13 @@
 -- handle changes in warpSkyfile.data_state.
--- Used for the modes tocleanedskyfile and topurgedskyfile
--- args are new data_state, warp_id, skycell_id and current expected state for warpRun
+-- Used for the modes tofullskyfile, tocleanedskyfile and topurgedskyfile
+-- args are new data_state, string for magic hook, warp_id, skycell_id and current expected state for warpRun
 UPDATE warpSkyfile
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
     SET 
     data_state = '%s'
+    -- set magicked hook %s
 WHERE
     warp_id = %lld
Index: trunk/ippTools/share/warptool_finish_run.sql
===================================================================
--- trunk/ippTools/share/warptool_finish_run.sql	(revision 26566)
+++ trunk/ippTools/share/warptool_finish_run.sql	(revision 26567)
@@ -1,5 +1,5 @@
 UPDATE warpRun
     SET state = 'full',
-    magicked = %d
+    magicked = %lld
 WHERE warp_id = %lld
 
Index: trunk/ippTools/share/warptool_finished_run_select.sql
===================================================================
--- trunk/ippTools/share/warptool_finished_run_select.sql	(revision 26566)
+++ trunk/ippTools/share/warptool_finished_run_select.sql	(revision 26567)
@@ -8,5 +8,5 @@
        warpSkyCellMap.warp_id as foo,
        warpSkyfile.warp_id as bar,
-       SUM(!warpSkyfile.magicked) = 0 as magicked
+       warpSkyfile.magicked as magicked
    FROM warpRun
    JOIN warpSkyCellMap
Index: trunk/ippTools/share/warptool_towarped.sql
===================================================================
--- trunk/ippTools/share/warptool_towarped.sql	(revision 26566)
+++ trunk/ippTools/share/warptool_towarped.sql	(revision 26567)
@@ -40,4 +40,6 @@
         AND warpSkyfile.tess_id IS NULL)
     OR (warpRun.state = 'update'
+        AND chipRun.magicked >= 0
+        AND warpSkyfile.fault = 0
         AND warpSkyfile.data_state = 'cleaned')
     )
Index: trunk/ippTools/src/camtool.c
===================================================================
--- trunk/ippTools/src/camtool.c	(revision 26566)
+++ trunk/ippTools/src/camtool.c	(revision 26567)
@@ -222,4 +222,5 @@
                     tess_id     ? tess_id   : row->tess_id,
                     end_stage   ? end_stage : row->end_stage,
+                    row->magicked,
                     note
         )) {
@@ -480,5 +481,6 @@
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
 
-    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
+//    Get this from the chipRun
+//    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
 
     // generate restrictions
@@ -523,4 +525,7 @@
         return false;
     }
+
+    bool status;
+    psS64 chip_magicked = psMetadataLookupS64(&status, output->data[0], "chip_magicked");
 
     camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]);
@@ -615,5 +620,6 @@
 
     // since there is only one exp per 'new' set camRun.state = 'full'
-    if (!pxcamRunSetState(config, row->cam_id, "full", magicked)) {
+    // propagate magicked state from chipRun
+    if (!pxcamRunSetState(config, row->cam_id, "full", chip_magicked)) {
         psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id);
         psFree(row);
Index: trunk/ippTools/src/chiptool.c
===================================================================
--- trunk/ippTools/src/chiptool.c	(revision 26566)
+++ trunk/ippTools/src/chiptool.c	(revision 26567)
@@ -720,4 +720,8 @@
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
     PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL);
+    if (psListLength(where->list) == 0) {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        return false;
+    }
         
     if (!state) {
@@ -1254,4 +1258,5 @@
                     chipRun->tess_id,
                     chipRun->end_stage,
+                    chipRun->magicked,
                     NULL    // note does not propagate
         )) {
@@ -1295,13 +1300,20 @@
     }
 
-    psString set_magic = "";
+    char *set_magicked_imfile = "";
+    char *set_magicked_run = "";
     if (!strcmp(data_state, "full")) {
-        // copy the magicked state from the input to the output when transitioning to full state
-        set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked";
-    }
-
-    // note only updates if chipRun.state = run_state
-    // XXX note that we have removed this constraint for now
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) {
+        // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will
+        // block warp processing until destreaking has been done
+        // otherwise copy magicked from the rawImfile
+        // Same thing for chipRun/rawExp
+        set_magicked_imfile = "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0 AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)";
+        set_magicked_run = "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), chipRun.magicked, rawExp.magicked)";
+    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
+        // if magicked is non-zero set it to -1
+        set_magicked_imfile = "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)";
+        set_magicked_run = "\n, chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)";
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) {
         psFree(query);
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1320,5 +1332,5 @@
 
     query = pxDataGet("chiptool_change_exp_state.sql");
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) {
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) {
         psFree(query);
         // rollback
Index: trunk/ippTools/src/chiptoolConfig.c
===================================================================
--- trunk/ippTools/src/chiptoolConfig.c	(revision 26566)
+++ trunk/ippTools/src/chiptoolConfig.c	(revision 26567)
@@ -154,5 +154,5 @@
     psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
-    psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", false);
+    psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", 0);
 
     // -processedimfile
Index: trunk/ippTools/src/difftool.c
===================================================================
--- trunk/ippTools/src/difftool.c	(revision 26566)
+++ trunk/ippTools/src/difftool.c	(revision 26567)
@@ -55,4 +55,5 @@
 static bool topurgedskyfileMode(pxConfig *config);
 static bool toscrubbedskyfileMode(pxConfig *config);
+static bool tofullskyfileMode(pxConfig *config);
 
 
@@ -94,7 +95,8 @@
         MODECASE(DIFFTOOL_MODE_EXPORTRUN,             exportrunMode);
         MODECASE(DIFFTOOL_MODE_IMPORTRUN,             importrunMode);
-        MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
-        MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE,    topurgedskyfileMode);
-        MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,  toscrubbedskyfileMode);
+        MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE,      tocleanedskyfileMode);
+        MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE,       topurgedskyfileMode);
+        MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,     toscrubbedskyfileMode);
+        MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE,         tofullskyfileMode);
 
         default:
@@ -2036,4 +2038,11 @@
   PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
 
+
+  psS64 magicked = 0;
+  if (!strcmp(data_state, "full")) {
+      PXOPT_LOOKUP_S64(set_magicked, config->args, "-magicked", 0, false);
+      magicked = set_magicked;
+  } 
+
   psString query = pxDataGet("difftool_change_skyfile_data_state.sql");
 
@@ -2043,7 +2052,18 @@
   }
 
+  psString set_magicked_skyfile = psStringCopy("");
+  psString set_magicked_run = psStringCopy("");
+  if (magicked != 0 && !strcmp(data_state, "full")) {
+    psStringAppend(&set_magicked_skyfile,  "\n , diffSkyfile.magicked = %" PRId64, magicked);
+    psStringAppend(&set_magicked_run,      "\n , diffRun.magicked = %" PRId64, magicked);
+
+  } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
+    psStringAppend(&set_magicked_skyfile, "\n, diffSkyfile.magicked = IF(diffSkyfile.magicked = 0, 0, -1)");
+    psStringAppend(&set_magicked_run, "\n, diffRun.magicked = IF(diffRun.magicked = 0, 0, -1)");
+  }
+
   // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c
 
-  if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, skycell_id)) {
+  if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, diff_id, skycell_id)) {
     psError(PS_ERR_UNKNOWN, false, "database error");
     // rollback
@@ -2052,10 +2072,12 @@
     }
     psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(set_magicked_skyfile);
     return(false);
   }
+  psFree(set_magicked_skyfile);
   psFree(query);
 
   query = pxDataGet("difftool_change_run_state.sql");
-  if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, data_state)) {
+  if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, diff_id, data_state)) {
     // rollback
     if (!psDBRollback(config->dbh)) {
@@ -2063,4 +2085,5 @@
     }
     psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(set_magicked_run);
     return(false);
   }
@@ -2068,6 +2091,8 @@
   if (!psDBCommit(config->dbh)) {
     psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(set_magicked_run);
     return(false);
   }
+  psFree(set_magicked_run);
 
   return(true);
@@ -2082,4 +2107,7 @@
 static bool toscrubbedskyfileMode(pxConfig *config) {
   return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed");
+}
+static bool tofullskyfileMode(pxConfig *config) {
+  return change_skyfile_data_state(config, "full", "update");
 }
 
Index: trunk/ippTools/src/difftool.h
===================================================================
--- trunk/ippTools/src/difftool.h	(revision 26566)
+++ trunk/ippTools/src/difftool.h	(revision 26567)
@@ -45,5 +45,6 @@
     DIFFTOOL_MODE_TOCLEANEDSKYFILE,
     DIFFTOOL_MODE_TOPURGEDSKYFILE,
-    DIFFTOOL_MODE_TOSCRUBBEDSKYFILE
+    DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,
+    DIFFTOOL_MODE_TOFULLSKYFILE
     
 } difftoolMode;
Index: trunk/ippTools/src/difftoolConfig.c
===================================================================
--- trunk/ippTools/src/difftoolConfig.c	(revision 26566)
+++ trunk/ippTools/src/difftoolConfig.c	(revision 26567)
@@ -122,5 +122,5 @@
     psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0,   "define hostname", 0);
     psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac",  0, "define %% of good pixels", NAN);
-    psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", false);
+    psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", 0);
 
     // -advance
@@ -289,4 +289,10 @@
     psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
 
+    // -tofullskyfile
+    psMetadata *tofullskyfileArgs = psMetadataAlloc();
+    psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0);
+    psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
+    psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", 0);
+
     // -exportrun
     psMetadata *exportrunArgs = psMetadataAlloc();
@@ -329,4 +335,5 @@
     PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs);
     PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs);
+    PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {
Index: trunk/ippTools/src/flatcorr.c
===================================================================
--- trunk/ippTools/src/flatcorr.c	(revision 26566)
+++ trunk/ippTools/src/flatcorr.c	(revision 26567)
@@ -535,4 +535,5 @@
 		row->tess_id,
 		"camera",
+                row->magicked,
                 NULL // note does not propragate
                 )) {
Index: trunk/ippTools/src/magicdstool.c
===================================================================
--- trunk/ippTools/src/magicdstool.c	(revision 26566)
+++ trunk/ippTools/src/magicdstool.c	(revision 26567)
@@ -44,5 +44,5 @@
 static bool tocleanupMode(pxConfig *config);
 
-static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state);
+static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state);
 
 # define MODECASE(caseName, func) \
@@ -326,8 +326,15 @@
     PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
 
+    // optional
+    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
+    psString setString = NULL;
+    if (set_label) {
+        psStringAppend(&setString, ", label = '%s'", set_label);
+    } 
+
     PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false);
     if (magic_ds_id) {
 
-        return setmagicDSRunState(config, magic_ds_id, NULL, state);
+        return setmagicDSRunState(config, magic_ds_id, setString, NULL, state);
 
     } else if (!strcmp(state, "full")) {
@@ -336,4 +343,5 @@
     }
     // we can transition by query as well
+
 
     psMetadata *where = psMetadataAlloc();
@@ -342,5 +350,4 @@
     PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
 
-
     if (psListLength(where->list) < 2) {
         psError(PS_ERR_UNKNOWN, true, "at least 2 search arguments are required");
@@ -353,5 +360,5 @@
         psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0);
     }
-    bool result = setmagicDSRunState(config, magic_ds_id, where, state);
+    bool result = setmagicDSRunState(config, magic_ds_id, setString, where, state);
     psFree(where);
 
@@ -445,5 +452,5 @@
 {
     // first query the magicDSRun to find the stage and the stage_id
-    psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
+    psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64;
 
     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
@@ -472,4 +479,5 @@
     psString stage= psMetadataLookupStr(NULL, row, "stage");
     psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
+    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
 
     ippStage stageNum = ippStringToStage(stage);
@@ -499,5 +507,5 @@
     }
 
-    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) {
+    if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -518,5 +526,5 @@
 {
     // first query the magicDSRun to find the stage and the stage_id
-    psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
+    psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64;
 
     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
@@ -545,4 +553,5 @@
     psString stage = psMetadataLookupStr(NULL, row, "stage");
     psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
+    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
 
     ippStage stageNum = ippStringToStage(stage);
@@ -570,5 +579,5 @@
         return false;
     }
-    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) {
+    if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -710,5 +719,5 @@
 
         // set magicDSRun.state to 'full'
-        if (!setmagicDSRunState(config, magic_ds_id, NULL, "full")) {
+        if (!setmagicDSRunState(config, magic_ds_id, NULL, NULL, "full")) {
             psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64,
                 magic_ds_id);
@@ -927,5 +936,5 @@
 }
 
-static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psMetadata *where, const char *state)
+static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psString extraSetStr, psMetadata *where, const char *state)
 {
     PS_ASSERT_PTR_NON_NULL(state, false);
@@ -937,5 +946,6 @@
     }
 
-    psString query = psStringCopy("UPDATE magicDSRun SET state = '%s'");
+    psString query = NULL;
+    psStringAppend(&query, "UPDATE magicDSRun SET state = '%s' %s\n", state, extraSetStr ? extraSetStr : "");
     if (magic_ds_id) {
         psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id);
@@ -949,5 +959,5 @@
     }
 
-    if (!p_psDBRunQueryF(config->dbh, query, state)) {
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false,
                 "failed to change state for magic_id %" PRId64, magic_ds_id);
Index: trunk/ippTools/src/magicdstoolConfig.c
===================================================================
--- trunk/ippTools/src/magicdstoolConfig.c	(revision 26566)
+++ trunk/ippTools/src/magicdstoolConfig.c	(revision 26567)
@@ -74,4 +74,5 @@
     psMetadata *updaterunArgs = psMetadataAlloc();
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL);
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage",     0, "define stage", NULL);
Index: trunk/ippTools/src/pxcam.c
===================================================================
--- trunk/ippTools/src/pxcam.c	(revision 26566)
+++ trunk/ippTools/src/pxcam.c	(revision 26567)
@@ -152,4 +152,5 @@
                     char *tess_id,
                     char *end_stage,
+                    psS64 magicked,
                     char *note)
 {
@@ -168,5 +169,5 @@
 
     // queue the exp
-    // XXX chip_id is being cast here work around psS64 have a different type
+    // XXX chip_id and magicked are being cast here work around psS64 have a different type
     // different on 32/64
     if (!p_psDBRunQueryF(config->dbh, query,
@@ -182,4 +183,5 @@
                 tess_id  ? tess_id  : "NULL",
                 end_stage ? end_stage : "NULL",
+                (long long) magicked,
                 note     ? note     : "NULL",
                 (long long)chip_id
Index: trunk/ippTools/src/pxcam.h
===================================================================
--- trunk/ippTools/src/pxcam.h	(revision 26566)
+++ trunk/ippTools/src/pxcam.h	(revision 26567)
@@ -40,4 +40,5 @@
                         char *tess_id,
                         char *end_stage,
+                        psS64 magicked,
                         char *note);
 
Index: trunk/ippTools/src/pxchip.c
===================================================================
--- trunk/ippTools/src/pxchip.c	(revision 26566)
+++ trunk/ippTools/src/pxchip.c	(revision 26567)
@@ -129,5 +129,5 @@
 }
 
-bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked)
+bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -140,5 +140,5 @@
     }
 
-    char *query = "UPDATE chipRun SET state = '%s', magicked = %d WHERE chip_id = %" PRId64;
+    char *query = "UPDATE chipRun SET state = '%s', magicked = %" PRId64 " WHERE chip_id = %" PRId64;
     if (!p_psDBRunQueryF(config->dbh, query, state, magicked, chip_id)) {
         psError(PS_ERR_UNKNOWN, false,
@@ -174,4 +174,9 @@
 
     psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'");
+
+    if (!strcmp(state, "cleaned") || !strcmp(state, "purged")) {
+        // if magicked is non-zero set it to -1
+        psStringAppend(&query, ", chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)");
+    }
 
     if (where && psListLength(where->list) > 0) {
@@ -219,4 +224,7 @@
         psStringAppend(&query, " %s", whereClause);
         psFree(whereClause);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        return false;
     }
 
Index: trunk/ippTools/src/pxchip.h
===================================================================
--- trunk/ippTools/src/pxchip.h	(revision 26566)
+++ trunk/ippTools/src/pxchip.h	(revision 26567)
@@ -27,5 +27,5 @@
 
 
-bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked);
+bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked);
 bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state);
 
Index: trunk/ippTools/src/warptool.c
===================================================================
--- trunk/ippTools/src/warptool.c	(revision 26566)
+++ trunk/ippTools/src/warptool.c	(revision 26567)
@@ -1646,7 +1646,16 @@
     }
 
-    // note only updates if warpRun.state = run_state
-    // XXX note that we have removed this constraint for now
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) {
+    char *set_magicked_skyfile = "";
+    char *set_magicked_run = "";
+    if (!strcmp(data_state, "full")) {
+        set_magicked_skyfile = "\n , warpSkyfile.magicked = IF((warpSkyfile.magicked < 0 AND chipRun.magicked = 0), warpSkyfile.magicked, chipRun.magicked)";
+        set_magicked_run = "\n,  warpRun.magicked = IF((warpRun.magicked < 0 AND chipRun.magicked = 0), warpRun.magicked, chipRun.magicked)";
+
+    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
+        set_magicked_skyfile = "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1)";
+        set_magicked_run = "\n, warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)";
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         // rollback
@@ -1660,5 +1669,5 @@
 
     query = pxDataGet("warptool_change_run_state.sql");
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, data_state)) {
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) {
         // rollback
         if (!psDBRollback(config->dbh)) {
Index: trunk/ippTools/src/warptoolConfig.c
===================================================================
--- trunk/ippTools/src/warptoolConfig.c	(revision 26566)
+++ trunk/ippTools/src/warptoolConfig.c	(revision 26567)
@@ -212,5 +212,5 @@
     psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
     psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
-    psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", false);
+    psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", 0);
     psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
@@ -310,5 +310,5 @@
     psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
 
-    // -toupdateskyfile
+    // -updateskyfile
     psMetadata *updateskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,    "warptool ID to update", 0);