Changeset 35747

Timestamp:

Jul 3, 2013, 1:43:50 PM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130509

Files:

• Nebulous (modified) (1 prop)
• Nebulous-Server (modified) (1 prop)
• PS-IPP-Config/lib/PS/IPP/Config.pm (modified) (1 diff)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (2 diffs)
• dbconfig/cam.md (modified) (1 prop)
• dbconfig/changes.txt (modified) (1 prop)
• ippMonitor (modified) (1 prop)
• ippMonitor/czartool/czartool/Nebulous.pm (modified) (1 diff)
• ippMonitor/czartool/czartool/Plotter.pm (modified) (1 diff)
• ippMonitor/czartool/czartool/czarconfig.xml (modified) (1 diff)
• ippMonitor/czartool/sql/insert_new_stsci_nodes_stsciwave2.sql (copied) (copied from trunk/ippMonitor/czartool/sql/insert_new_stsci_nodes_stsciwave2.sql )
• ippMonitor/def/diffProcessedSkyfile_Images.d (modified) (2 diffs)
• ippMonitor/def/newsimplePlotStaticskyImage.d (copied) (copied from trunk/ippMonitor/def/newsimplePlotStaticskyImage.d )
• ippScripts/scripts/background_chip.pl (modified) (4 diffs)
• ippScripts/scripts/background_warp.pl (modified) (1 diff)
• ippScripts/scripts/camera_exp.pl (modified) (13 diffs)
• ippScripts/scripts/chip_imfile.pl (modified) (4 diffs, 1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/warp_skycell.pl (modified) (3 diffs, 1 prop)
• ippTools/share (modified) (1 prop)
• ippTools/share/bgtool_advancechip.sql (modified) (1 diff)
• ippTools/share/bgtool_definewarp.sql (modified) (1 diff)
• ippTools/share/bgtool_tochip.sql (modified) (3 diffs)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 prop)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/bgtool.c (modified) (1 diff)
• ippTools/src/camtool.c (modified) (1 prop)
• ippTools/src/camtoolConfig.c (modified) (1 prop)
• ippTools/src/chiptool.c (modified) (1 diff)
• ippTools/src/magictool.c (modified) (1 prop)
• ippTools/src/pstamptool.c (modified) (2 diffs)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff)
• ppImage/src (modified) (1 prop)
• ppImage/src/Makefile.am (modified) (1 diff)
• ppImage/src/burnparams.h (copied) (copied from trunk/ppImage/src/burnparams.h )
• ppImage/src/burntool.h (copied) (copied from trunk/ppImage/src/burntool.h )
• ppImage/src/ppImage.h (modified) (2 diffs)
• ppImage/src/ppImageArguments.c (modified) (1 diff)
• ppImage/src/ppImageAuxiliaryMask.c (modified) (1 diff)
• ppImage/src/ppImageBurntoolApply.c (copied) (copied from trunk/ppImage/src/ppImageBurntoolApply.c )
• ppImage/src/ppImageDetrendReadout.c (modified) (2 diffs)
• ppImage/src/ppImageDetrendRecord.c (modified) (1 diff)
• ppImage/src/ppImageOptions.c (modified) (2 diffs)
• ppImage/src/ppImageParseCamera.c (modified) (1 diff)
• psastro/src/psastro.h (modified) (1 diff)
• psastro/src/psastroAnalysis.c (modified) (3 diffs)
• psastro/src/psastroArguments.c (modified) (3 diffs)
• psastro/src/psastroChooseRefstars.c (modified) (2 diffs)
• psastro/src/psastroExtractAnalysis.c (modified) (1 diff)
• psconfig (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/pstamp_job_run.pl (modified) (1 diff)
• pstamp/src/ppstampMakeStamp.c (modified) (4 diffs)
• psvideophot (modified) (1 prop)
• pswarp/src/pswarpDefineLayout.c (modified) (1 diff)
• pswarp/src/pswarpLoadAstrometry.c (modified) (3 diffs)
• pswarp/src/pswarpLoopBackground.c (modified) (1 diff)
• tools/cfitsio_tools (copied) (copied from trunk/tools/cfitsio_tools )
• tools/cfitsio_tools/Makefile (copied) (copied from trunk/tools/cfitsio_tools/Makefile )
• tools/cfitsio_tools/immath3d.c (copied) (copied from trunk/tools/cfitsio_tools/immath3d.c )
• tools/runchipimfile.pl (modified) (1 diff)
• tools/smf2ct.pl (copied) (copied from trunk/tools/smf2ct.pl )

branches/eam_branches/ipp-20130509/PS-IPP-Config/lib/PS/IPP/Config.pm

@@ -730 +730 @@
 }
 
+sub file_rename {
+    my $self = shift;
+    my $old_name = shift;
+    my $name = shift;
+
+    my $scheme = file_scheme($old_name);
+    my $newscheme = file_scheme($name);
+    if ($scheme and $scheme eq 'neb') {
+        if (!$newscheme or $newscheme ne 'neb') {
+            carp "cannot rename nebulous file to non nebulous path";
+            return undef;
+        }
+        my $neb = $self->nebulous;
+        if ($neb->storage_object_exists($old_name)) {
+            eval {
+                $neb->move($old_name, $name);
+            };
+            if ($@) {
+                carp "nebulous move failed for $old_name";
+                return undef;
+            }
+        } else {
+            carp "$old_name not found";
+            return undef;
+        }
+    } else {
+        if ($newscheme and $newscheme eq 'neb') {
+            carp "cannot rename non-nebulous file to nebulous path";
+            return undef;
+        }
+        if ($self->file_exists($old_name)) {
+            my $resolved = $self->file_resolve($old_name);
+            my $resolved_new = $self->file_resolve($name);
+            if (!rename($resolved, $resolved_new)) {
+                carp "Failed to rename $resolved $resolved_new";
+                return undef;
+            }
+        } else {
+            carp "$old_name not found";
+            return undef;
+        }
+    }
+    return $name;
+}
+
 # redirect stderr and stdout streams to log file. If file already exists rename it and record
 # the name in the new log file.

branches/eam_branches/ipp-20130509/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -574 +574 @@
         $command .= " -dateobs_end $dateobs_end" if $dateobs_end;
     } elsif ($req_type ne "byid") {
-        $command .= " -mjd_min $mjd_min" if $mjd_min;
-        $command .= " -mjd_max $mjd_max" if $mjd_max;
+        $command .= " -mjd_obs_begin $mjd_min" if $mjd_min;
+        $command .= " -mjd_obs_end $mjd_max" if $mjd_max;
     }
 
@@ -1025 +1025 @@
     } else {
         $using_camtool = 1;
-        my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
-           # NOTE: we are applying the data_group to the camera run.
-           # If we're looking for chip stage images there is no guarentee that 
-           # chipRun.data_group eq camRun.data_group. In practice this is almost
-           # always the case. If this turns out to be a problem we can defer
-           # the data_group test to when we look up the chipProcessedImfiles
-           if ($data_group) {
-               $command .= " -data_group $data_group";
-           } else {
-               $command .= $default_data_groups;
-           }
-           $command .= " -destreaked" if $need_magic;
+        $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
+        # NOTE: we are applying the data_group to the camera run.
+        # If we're looking for chip stage images there is no guarentee that 
+        # chipRun.data_group eq camRun.data_group. In practice this is almost
+        # always the case. If this turns out to be a problem we can defer
+        # the data_group test to when we look up the chipProcessedImfiles
+        if ($data_group) {
+            $command .= " -data_group $data_group";
+        } else {
+            $command .= $default_data_groups;
+        }
+        $command .= " -destreaked" if $need_magic;
     }
     $command .= " -ra $ra -decl $dec -radius $search_radius";

branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/Nebulous.pm

@@ -113 +113 @@
         } elsif (($line =~ m/(ipp[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)||
                ($line =~ m/(ippb[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) ||
-               ($line =~ m/(stsci0[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
+               ($line =~ m/(stsci[0-1][0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
 
             $self->{_totalHosts}++;

branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/Plotter.pm

@@ -707 +707 @@
     if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
     print GP
-        "set term $self->{_outputFormat} size 1280,1024;" .
+        "set term $self->{_outputFormat} size 2560,1024;" .
         "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" .
         "set style fill solid 1.00 border -1;" .

branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/czarconfig.xml

@@ -47 +47 @@
     <serverinterval>20 MINUTE</serverinterval>
     <!-- whitespace-separated server list for servers we want roboczar to check --> 
-    <interestedservers>stdscience distribution summitcopy registration pstamp stack publishing</interestedservers>
+    <interestedservers>stdscience distribution summitcopy registration stack publishing pstamp </interestedservers>
     <!-- <interestedservers>pstamp</interestedservers> -->
   </roboczar>

branches/eam_branches/ipp-20130509/ippMonitor/def/diffProcessedSkyfile_Images.d

@@ -3 +3 @@
 FILE  diffProcessedSkyfile_Images.php
 MENU  ipp.stack.dat
+
+UNRESTRICTED AND 0>1
+RESTRICT_COUNT AND 0>1
 
 WHERE diffRun.diff_id             = diffSkyfile.diff_id
@@ -13 +16 @@
 
 # if no query restrictions are supplied, we want to limit the query or it is extremely long running:
-UNRESTRICTED AND diffRun.diff_id = 0
+# UNRESTRICTED AND diffRun.diff_id = 0
 
 # XXX need to get camera from lookup

branches/eam_branches/ipp-20130509/ippScripts/scripts/background_chip.pl

@@ -159 +159 @@
 }
 
-# XXX: M31 TEST turn off restoring pattern correction
-$apply_pattern = 0;
-
 # Set up files
 &my_die("Couldn't find input file: $in_image\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_image);
@@ -170 +167 @@
 
 $ipprc->outroot_prepare($outroot);
-#my $out_image = $ipprc->filename("PPBACKGROUND.OUTPUT", $outroot, $class_id);
-#my $out_mask = $ipprc->filename("PPBACKGROUND.OUTPUT.MASK", $outroot, $class_id);
-#my $out_stats = $ipprc->filename("PPBACKGROUND.STATS", $outroot, $class_id);
-#my $out_config = $ipprc->filename("PPBACKGROUND.CONFIG", $outroot, $class_id);
-#my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id);
 
 my $out_image = prepare_output("PPBACKGROUND.OUTPUT", $outroot, $class_id, 1);
@@ -183 +175 @@
 my $traceDest = prepare_output("TRACE.IMFILE", $outroot, $class_id, 1);
 
-my $auxmask;
-
-my $apply_auxiliary_mask = 1;
-# XXX Temporary lookup of auxiliary mask file from static location.
-# At some point I'll convert this to register the masks in the database and use detselect
-if ($apply_auxiliary_mask) {
-    my @auxmask_start_date = qw(
-    0000
-    5178
-    5218
-    5403
-    5424
-    5455
-    5485
-    5512
-    5545
-    5770
-    5797
-    5825
-    5854
-    5881
-    6138
-    6153
-    6197
-    6214
-    );
-    my @auxmask_end_date = qw(
-    5177
-    5217
-    5402
-    5423
-    5454
-    5484
-    5511
-    5544
-    5769
-    5796
-    5824
-    5853
-    5880
-    6137
-    6152
-    6196
-    6213
-    9999
-    );
-
-    my $outroot_base = basename($outroot);
-
-    # Assume that outroot base begins with exp_name: fix by passing in exp_name
-    my $mjd = substr($outroot_base, 1, 4);
-    if ($mjd =~ /\D/) {
-            my $error_code = 1;
-            &my_die("unexpected outroot base: $outroot_base: $error_code", $chip_bg_id, $class_id, $error_code);
-    }
-
-    my $i = 0;
-    while ($auxmask_end_date[$i] < $mjd) {
-        $i++;
-    }
-
-    my $auxmask_base = "/data/ipp064.0/bills/m31-masks/www.usm.uni-muenchen.de/people/arri/PAndromeda/masks_fits/mask";
-
-    $auxmask = sprintf "%s_%04d-%04d_%s.fits", $auxmask_base, $auxmask_start_date[$i], $auxmask_end_date[$i], $class_id;
-    print "Auxililary mask file is $auxmask\n";
-}
-
-
-
-# Run ppImage
+
+# Run ppBackground
 unless ($no_op) {
     my $command  = "$ppBackground $outroot";
@@ -258 +182 @@
     $command .= " -mask $in_mask";
     $command .= " -variance $in_wt";
-    $command .= " -auxmask $auxmask" if $auxmask;
     $command .= " -stats $out_stats" if $do_stats;
     $command .= " -background $in_bg" if $apply_bg;

branches/eam_branches/ipp-20130509/ippScripts/scripts/background_warp.pl

@@ -191 +191 @@
     $command .= " $outroot $skyFile";
     $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
+    # XXX: change fits type based on recipe
+    $command .= " -R PSWARP.OUTPUT FITS.TYPE NONE";
     $command .= " -recipe PSWARP $recipe_pswarp";
     $command .= " -recipe PPSTATS WARPSTATS";

branches/eam_branches/ipp-20130509/ippScripts/scripts/camera_exp.pl

@@ -53 +53 @@
     'skip-binned'       => \$skip_binned, # override recipe - don't create binned images
     'skip-refmask'      => \$skip_masks, # override recipe - don't create refmask
-    'bkg-only'          => \$bkg_only,  # override recipe - only do background continuity
+#    'bkg-only'          => \$bkg_only,  # override recipe - only do background continuity
     'verbose'           => \$verbose,   # Print to stdout
     'no-update'         => \$no_update, # Update the database?
@@ -80 +80 @@
 my $replicateOutputs = 1;
 
+$ipprc->outroot_prepare($outroot);
+
 my $logDest;
 my $traceDest;
@@ -114 +116 @@
 my $bkg_recipe     = 'PPIMAGE_BKGCONT'; # Add to reduction?
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $recipeData;
+{
+    # Get the PSASTRO recipe
+    my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
+    }
+    $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
+}
 
 my $cmdflags;
@@ -158 +174 @@
     chomp $cmdflags;
 
-    { # Determine if FWHM is too large to bother continuing.
-        my $maxFWHM = 0;
-      # XXX: temporary hack if this is a CNP exposure set maxFWHM to a large
-      # value.
-      # Before the next tag we will put this into a recipe
-      if ($outroot =~ /CNP/) {
-            $maxFWHM = 25;
-      } else {
-        my $command = "$ppConfigDump -camera $camera -get-key MAX_ALLOWED_FWHM";
-        my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
-        }
-        if (@$stdout_buf) {
-            my $cameraConfig = $mdcParser->parse(join "", @$stdout_buf) or
-                &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
-            $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM');
-        }
-      }
-        if ($maxFWHM) {
-            my $expFWHM;
-            ($expFWHM) = $cmdflags =~ /-fwhm_major (\d+)/;      
-            
-            if ($expFWHM > $maxFWHM) {
-                print "Setting quality to 4007 due to large FWHM: exposure: $expFWHM  maximum: $maxFWHM\n";
-                $cmdflags .= " -quality 4007 "; # This corresponds to PSASTRO_ERR_DATA
-                $no_op = 1;
-            }
-        }
-    }
-}
-
-
-my $produceMasks;               # Produce masks?
+    # Determine if FWHM is too large to bother continuing.
+    my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM');
+    if ($maxFWHM) {
+        my $expFWHM;
+        ($expFWHM) = $cmdflags =~ /-fwhm_major (\d+)/;  
+        
+        if ($expFWHM > $maxFWHM) {
+            print "Setting quality to 4007 due to large FWHM: exposure: $expFWHM  maximum: $maxFWHM\n";
+            $cmdflags .= " -quality 4007 "; # This corresponds to PSASTRO_ERR_DATA
+            $no_op = 1;
+        }
+    }
+}
+
+
+my $do_masks;               # Produce masks?
 if (!$skip_masks) {
-    # Get the PSASTRO recipe
-    my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
-    }
-    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
-
-    $produceMasks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
+    $do_masks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
 }
 
@@ -220 +205 @@
 my ($list5File, $list5Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b5.list.XXXX", UNLINK => !$save_temps ); # For background models
 
-# XXX we perform astrometry iff photometry output exists
+
+my $do_stats;
+my $do_bkg;
+my $do_jpegs;
+my $fpaStats; 
+my $psastroInputArg;
+if ($run_state eq 'new') {
+    $do_stats = 1;
+    $do_bkg = 1;
+    $do_jpegs = !$skip_binned;
+    $fpaStats = prepare_output("PSASTRO.STATS",      $outroot, undef, 1);
+    $psastroInputArg = " -list $list3Name";
+} else {
+    # for $run_state eq 'update' we only rebuild the masks using psastro
+    $do_stats = 0;
+    $do_jpegs = 0;
+    $do_bkg = 1;        # we could skip this step if camProcessedExp.backgroun_model is non zero
+
+    if (!$do_masks) {
+        &my_die("run_state is update but do_masks is F. I have nothing to do!!", $cam_id, $PS_EXIT_UNKNOWN_ERROR)
+    }
+
+    # the input to psastro is the original PSASTRO.OUTPUT
+    
+    # Check for file with rule PSASTRO.OUTPUT.ORGINAL
+    # This file will exist if we attempted to update this camRun before but faulted
+    # XXX: make sure cleanup deals with these files
+    my $inputObjects = $ipprc->filename("PSASTRO.OUTPUT.ORIGINAL", $outroot, undef);
+    if (!$ipprc->file_exists($inputObjects)) {
+        # not found so original file should still be in place. 
+        # Rename it.
+        my $originalObjects = $ipprc->filename("PSASTRO.OUTPUT", $outroot, undef);
+        if ($ipprc->file_exists($originalObjects)) {
+
+            unless ($ipprc->file_rename($originalObjects, $inputObjects)) {
+                &my_die("failed to rename $originalObjects to $inputObjects", $cam_id, $PS_EXIT_UNKNOWN_ERROR);
+            }
+
+            # ok ready to go
+        } else {
+            print STDERR "failed to find input objects to update\n";
+            print STDERR "Original file name: $originalObjects\n";
+            print STDERR "Saved file name:    $inputObjects\n";
+
+            &my_die("Cannot proceed.", $cam_id, $PS_EXIT_DATA_ERROR);
+
+            # XXX: actually we could use the chip stage cmfs as inputs and turn on astrometry 
+            # ... except that somebody (me) got the bright idea to save space and 
+            # changed chip stage cleanup to remove them. So we're kind of stuck.
+        }
+    }
+    $psastroInputArg = " -file $inputObjects -skipastro";
+}
+    
+
 my @outMasks;                   # Names of output masks
 my @bkg_models;                 # Names of output background models
 foreach my $file (@$files) {
+    # we perform astrometry iff photometry output exists
     next if $file->{quality} != 0;
+
     # use the path_base as OUTPUT root and convert the filenames with ipprc->filename:
     my $class_id = $file->{class_id};
@@ -239 +280 @@
     print $list5File ($ipprc->filename("PSPHOT.BACKMDL", $file->{path_base}, $class_id) . "\n");
 
-    push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $produceMasks;
-    push @bkg_models, prepare_output("PPIMAGE.BACKMDL", $outroot, $class_id, 1);
+    push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $do_masks;
+    push @bkg_models, prepare_output("PPIMAGE.BACKMDL", $outroot, $class_id, 1) if $do_bkg;
 }
 close $list1File;
@@ -247 +288 @@
 close $list4File;
 close $list5File;
-# Output products
-$ipprc->outroot_prepare($outroot);
+
+# Prepare the Output products
 
 # the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs
-my $jpeg1      = prepare_output("PPIMAGE.JPEG1",      $outroot, undef, 1);
-my $jpeg2      = prepare_output("PPIMAGE.JPEG2",      $outroot, undef, 1);
-my $fpaObjects = prepare_output("PSASTRO.OUTPUT",     $outroot, undef, 1);
-my $configuration = prepare_output("PSASTRO.CONFIG",  $outroot, undef, 1);
-
-my $do_stats;
-my $fpaStats; 
+my $jpeg1      = prepare_output("PPIMAGE.JPEG1",      $outroot, undef, 1) if $do_jpegs;
+my $jpeg2      = prepare_output("PPIMAGE.JPEG2",      $outroot, undef, 1) if $do_jpegs;
+my $fpaObjects = prepare_output("PSASTRO.OUTPUT",     $outroot, undef, 1) if !$bkg_only;
+
+my $configuration;
 if ($run_state eq 'new') {
-    $do_stats = 1;
-    $fpaStats = prepare_output("PSASTRO.STATS",      $outroot, undef, 1);
+    $configuration = prepare_output("PSASTRO.CONFIG",  $outroot, undef, 1);
 } else {
-    $do_stats = 0;
-}
+    # Do not use the original recipes for updates because they might contain the 
+    # recipe values that caused the masks to get fouled up in the first place
+    # $configuration = $ipprc->filename("PSASTRO.CONFIG",  $outroot, undef);
+}
+
 
 unless ($no_op) {
@@ -269 +310 @@
     ## build the output JPEG images first so we get them even if the astrometry fails
 
-    # Make the jpeg for binning 1
-    if (!$skip_binned and ($run_state eq 'new')) {
+    if ($do_jpegs) {
+        # Make the jpeg for binning 1
         my $command = "$ppImage -list $list1Name $outroot"; # Command to run
         $command .= " -recipe PPIMAGE $recipe1";
@@ -282 +323 @@
         }
         check_output($jpeg1, $replicateOutputs);
-    }
-
-    # Make the jpeg for binning 2
-    if (!$skip_binned and ($run_state eq 'new')) {
-        my $command = "$ppImage -list $list2Name $outroot"; # Command to run
+
+        # Make the jpeg for binning 2
+
+        $command = "$ppImage -list $list2Name $outroot"; # Command to run
         $command .= " -recipe PPIMAGE $recipe2";
         $command .= " -dbname $dbname" if defined $dbname;
 
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
         unless ($success) {
@@ -302 +342 @@
         # run psastro on the chipObjects, producing fpaObjects
         my $command;
-        $command  = "$psastro";
-        $command .= " -list $list3Name";
-        $command .= " -masklist $list4Name" if $produceMasks;
+        $command  = $psastro;
+        $command .= " $psastroInputArg";
+        $command .= " -masklist $list4Name" if $do_masks;
         $command .= " $outroot";
         $command .= " -recipe PSASTRO $recipe_psastro";
-        $command .= " -Db PSASTRO:REFSTAR_MASK F" if $skip_masks;
+        $command .= " -Db PSASTRO:REFSTAR_MASK F" if !$do_masks;
         $command .= " -tracedest $traceDest -log $logDest";
         $command .= " -dbname $dbname" if defined $dbname;
@@ -314 +354 @@
             $command .= " -dumpconfig $configuration";
         } elsif ($run_state eq 'update') {
-            $command .= " -ipprc $configuration";
+            $command .= " -ipprc $configuration" if $configuration;
         } else {
             &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR);
@@ -362 +402 @@
         }
     }
+
     # Construct FPA continuity corrected background images
     if ($camera =~ /ISP/) {
         print "Skipping FPA continuity corrected background images for ISP\n";
-    } else {
+    } elsif ($do_bkg) {
         my $command;
         $command = "$ppImage";
@@ -381 +422 @@
             check_output($bkgModel, $replicateOutputs);
         }
-        $cmdflags = $cmdflags . " -background_model 1 ";
+        if ($run_state eq 'new') {
+            $cmdflags .= " -background_model 1 ";
+        } else {
+            my $command = "camtool -updateprocessedexp -set_background_model 1 -fault 0 -cam_id $cam_id";
+            $command .= " -dbname $dbname" if $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to updateprocessedexp to set background_model: $error_code", $cam_id, $error_code);
+            }
+        }
     }
 }

branches/eam_branches/ipp-20130509/ippScripts/scripts/chip_imfile.pl

@@ -200 +200 @@
 
     ## get the ppImage recipe for this camera and CHIP reduction
-    $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe_ppImage -";
+    $command = "$ppConfigDump -dump-recipe PPIMAGE -recipe PPIMAGE $recipe_ppImage -";
+    if ($dump_config) {
+        # use this camera's recipes
+        $command .= " -camera $camera";
+    } else {
+        # get recipes as set in config dump file
+        $command .= " -ipprc $configuration";
+    }
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 0);
@@ -224 +231 @@
     }
 
-
-    ## XXX make the feature more general?
+    my $no_compress_image = metadataLookupBool($recipeData, 'NO.COMPRESS');
+
     my $useDeburnedImage = metadataLookupBool($recipeData, 'USE.DEBURNED.IMAGE');
+    my $ppImageApplyBurntool = metadataLookupBool($recipeData, 'APPLY.BURNTOOL');
+    my $burntoolArguments;
 
     if ($useDeburnedImage) {
@@ -327 +336 @@
         }
 
-        ## We now know that we have an image that has been burntooled.
-        my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.deburned.XXXX",
+        # if recipe option PPIMAGE:APPLY.BURNTOOL is set and the burntool information is stored in
+        # an external table (usual case), check the existence of that table file and set $burntoolArguments.
+        # If burntool state does not match, say if the burntool data is stored in a fits extension in
+        # the rawImfile, use the burntool program.
+        if ($ppImageApplyBurntool && ($burntoolState == -1 * $burntoolStateGood)) {
+            my $burntoolTable_uri = $uri;
+            $burntoolTable_uri =~ s/fits$/burn.tbl/;
+            my $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+            if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
+                my $repair_cmd = "$nebrepair $burntoolTable_uri";
+                my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) 
+                    = run(command => $repair_cmd, verbose => $verbose);
+                unless ($repair_success) {
+                    &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id,
+                        $PS_EXIT_SYS_ERROR);
+                }
+                $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+            }
+            unless ($ipprc->file_exists($burntoolTable_uri)) {
+                &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, 
+                    $PS_EXIT_SYS_ERROR);
+            }
+            $burntoolArguments = " -burntool $burntoolTable_uri";
+        } else {
+            ## use external burntool program 
+
+            ## We now know that we have an image that has been burntooled.
+            my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.deburned.XXXX",
                                               UNLINK => !$save_temps, SUFFIX => '.fits' );
 
-        # get the UNIX version of the (possible) neb: or path: filename
-        my $uriReal = $ipprc->file_resolve( $uri );
-        # Catch errors here
-        if (!$uriReal) {
-            my $repair_cmd = "$nebrepair $uri";
-            my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
-            unless ($repair_success) {
-                &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-            }
-            $uriReal = $ipprc->file_resolve( $uri );
-        }
-            
-        &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
-
-        # funpack into the temp file.
-        my $funpack_cmd = "$funpack -S $uriReal > $tempName";
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
-        unless ($success) {
-            # Catch errors here
-            my $repair_cmd = "$nebrepair $uri";
-            my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
-            unless ($repair_success) {
-                &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-            }
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
-        }
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-        }
-
-        ## Construct commands to apply the pre-calculated burntool trailfits to the pixel data.
-        my ($burntoolTable_uri, $burntoolTable_uriReal);
-        my $burntool_cmd = "$burntool ";
-
-        if ($burntoolState == -1 * $burntoolStateGood) {   # Burntool information stored in an external table.
-            $burntoolTable_uri = $uri;
-            $burntoolTable_uri =~ s/fits$/burn.tbl/;
-            $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
-            if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
-                my $repair_cmd = "$nebrepair $burntoolTable_uri";
-                my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
-                unless ($repair_success) {
-                    &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-                }
-                $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
-            }
-            unless ($ipprc->file_exists($burntoolTable_uri)) {
-                # Catch errors here
-                &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR);
-            }
-
-            $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} persist=t apply=t";
-        }
-        elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table.
-            $burntool_cmd .= "$tempName persist=t apply=t";
-        }
-        else {
-            &my_die("Image data not properly burntooled, impossible state", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-        }
-
-        ## Run burntool to change the pixels of tempfile, and repoint $uri to that file.
-        ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
-            run(command => $burntool_cmd, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform burntool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-        }
-
-        $uri = $tempName;
-        unless ($ipprc->file_exists($uri)) {
-            &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-        }
-
-#       print STDERR "$uri $uriReal $tempName $burntoolTable_uri $burntoolTable_uriReal $burntool_cmd $save_temps\n";
-#       exit(100);
-    }
-
-    # apply the 'tiltystreak' operation, if desired
+            # get the UNIX version of the (possible) neb: or path: filename
+            my $uriReal = $ipprc->file_resolve( $uri );
+                # Catch errors here
+            if (!$uriReal) {
+                my $repair_cmd = "$nebrepair $uri";
+                my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+                unless ($repair_success) {
+                    &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+                }
+                $uriReal = $ipprc->file_resolve( $uri );
+            }
+                
+            &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
+
+            # funpack into the temp file.
+            my $funpack_cmd = "$funpack -S $uriReal > $tempName";
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
+            unless ($success) {
+                # Catch errors here
+                my $repair_cmd = "$nebrepair $uri";
+                my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+                unless ($repair_success) {
+                    &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+                }
+                ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
+            }
+            unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+
+            ## Construct commands to apply the pre-calculated burntool trailfits to the pixel data.
+            my ($burntoolTable_uri, $burntoolTable_uriReal);
+            my $burntool_cmd = "$burntool ";
+
+            if ($burntoolState == -1 * $burntoolStateGood) {   # Burntool information stored in an external table.
+                $burntoolTable_uri = $uri;
+                $burntoolTable_uri =~ s/fits$/burn.tbl/;
+                $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+                if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
+                    my $repair_cmd = "$nebrepair $burntoolTable_uri";
+                    my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+                    unless ($repair_success) {
+                        &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+                    }
+                    $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+                }
+                unless ($ipprc->file_exists($burntoolTable_uri)) {
+                    # Catch errors here
+                    &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR);
+                }
+
+                $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} persist=t apply=t";
+            }
+            elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table.
+                $burntool_cmd .= "$tempName persist=t apply=t";
+            }
+            else {
+                &my_die("Image data not properly burntooled, impossible state", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+
+            ## Run burntool to change the pixels of tempfile, and repoint $uri to that file.
+            ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
+                run(command => $burntool_cmd, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform burntool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+
+            $uri = $tempName;
+            unless ($ipprc->file_exists($uri)) {
+                &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+
+            if ($dump_config) {
+                # tell ppImage to not apply burntool since we have done it here
+                $burntoolArguments = ' -Db PPIMAGE:APPLY.BURNTOOL F';
+            }
+    #       print STDERR "$uri $uriReal $tempName $burntoolTable_uri $burntoolTable_uriReal $burntool_cmd $save_temps\n";
+    #       exit(100);
+
+        }
+    }
+
     my $tiltystreakApply = metadataLookupBool($recipeData, 'TILTYSTREAK.APPLY');
     if ($tiltystreakApply) {
-
-        my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS');
-        my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id");
-        if ($tiltystreakClassApply) {
-
-            my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id");
-            if ($tiltystreakClassOptions eq "none") {
-                $tiltystreakClassOptions = "";
-            }
-
-            my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
-
-            # XXX make these optional (must be built by psbuild as well...)
-            my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-            my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-
-            # create an temporary output file:
-            my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps );
-
-            print "uri: $uri\n";
-
-            # get the UNIX version of the (possible) neb: or path: filename
-            my $uriReal = $ipprc->file_resolve( $uri );
-
-            print "uriReal: $uriReal\n";
-
-            # unpack the data (is a NOP if not compressed)
-            $command = "$funpack -S $uriReal > $tempName";
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-            }
-
-            # run tiltystreak
-            $command = "$tiltystreak $tempName $tiltystreakClassOptions";
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-            }
-
-            # supply the output file as the new input file
-            $uri = $tempName;
-        }
+        # NOTE: application of TILTYSTREAK.APPLY was here. Removed after r35685
+        &my_die("Unable to perform tiltystreak: code has been removed", $exp_id, $chip_id, $class_id,
+            $PS_EXIT_PROG_ERROR);
     }
 
@@ -479 +477 @@
         $command .= " -Db PPIMAGE:BIN1.FITS FALSE -Db PPIMAGE:BIN2.FITS FALSE";
     }
+    if ($no_compress_image) {
+        $command .= " -R PPIMAGE.CHIP FITS.TYPE NONE";
+    }
+    $command .= $burntoolArguments if $burntoolArguments;
     $command .= " -threads $threads" if defined $threads;
     $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;

branches/eam_branches/ipp-20130509/ippScripts/scripts/warp_skycell.pl

@@ -129 +129 @@
 my $astromSource;               # The astrometry source
 my $doBackground;               # Do we want to make background models?
+my $noCompression;
 {
     my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -";
@@ -141 +142 @@
     $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
     $doBackground = metadataLookupBool($metadata, 'BACKGROUND.MODEL');    
+    $noCompression = metadataLookupBool($metadata, 'NO.COMPRESS');
 }
 
@@ -299 +301 @@
     $command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL";
     $command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID";
+    $command .= " -R PSWARP.OUTPUT FITS.TYPE NONE" if $noCompression;
     $command .= " -recipe PSWARP $recipe_pswarp";
     $command .= " -tracedest $traceDest -log $logDest";

branches/eam_branches/ipp-20130509/ippTools/share/bgtool_advancechip.sql

@@ -3 +3 @@
     chipBackgroundImfile.magicked
 FROM chipBackgroundRun
-JOIN chipProcessedImfile USING(chip_id)
-LEFT JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
+JOIN camRun using(cam_id)
+JOIN chipProcessedImfile ON chipBackgroundRun.chip_id = chipProcessedImfile.chip_id
+JOIN chipRun AS bgsub_chipRun on camRun.chip_id = bgsub_chipRun.chip_id
+JOIN chipProcessedImfile AS bgsub_chipProcessedImfile 
+    ON (bgsub_chipRun.chip_id = bgsub_chipProcessedImfile.chip_id 
+        AND bgsub_chipProcessedImfile.class_id = chipProcessedImfile.class_id)
+LEFT JOIN chipBackgroundImfile 
+    ON (chipBackgroundRun.chip_bg_id = chipBackgroundImfile.chip_bg_id
+        AND chipBackgroundImfile.class_id = chipProcessedImfile.class_id)
 WHERE chipBackgroundRun.state = 'new'
     AND chipProcessedImfile.quality = 0
     AND chipProcessedImfile.fault = 0
+    AND bgsub_chipProcessedImfile.quality = 0
 -- WHERE hook %s
-GROUP BY chip_bg_id
+GROUP BY chipBackgroundRun.chip_bg_id
 HAVING
     COUNT(chipBackgroundImfile.class_id) = COUNT(chipProcessedImfile.class_id)

branches/eam_branches/ipp-20130509/ippTools/share/bgtool_definewarp.sql

@@ -15 +15 @@
 JOIN chipBackgroundRun ON chipRun.chip_id = chipBackgroundRun.chip_id 
     AND (chipBackgroundRun.cam_id = camRun.cam_id OR chipBackgroundRun.cam_id = 0)
-LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.warp_bg_id -- label hook %s
+LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.chip_bg_id -- label hook %s
 WHERE chipBackgroundRun.state = 'full'
     AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- need warp to have completed so warpSkyCellMap is populated

branches/eam_branches/ipp-20130509/ippTools/share/bgtool_tochip.sql

@@ -1 +1 @@
 SELECT
     chipBackgroundRun.*,
-    class_id,
+    chipProcessedImfile.class_id,
     rawExp.exp_tag,
     rawExp.camera
@@ -8 +8 @@
 JOIN rawExp USING(exp_id)
 JOIN chipProcessedImfile USING(chip_id)
-LEFT JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
+JOIN camRun using(cam_id)
+JOIN chipRun AS bgsub_chipRun on camRun.chip_id = bgsub_chipRun.chip_id
+JOIN chipProcessedImfile AS bgsub_chipProcessedImfile 
+    ON (bgsub_chipRun.chip_id = bgsub_chipProcessedImfile.chip_id 
+        AND bgsub_chipProcessedImfile.class_id = chipProcessedImfile.class_id)
+LEFT JOIN chipBackgroundImfile  
+    ON (chipBackgroundRun.chip_bg_id = chipBackgroundImfile.chip_bg_id 
+        AND chipBackgroundImfile.class_id = chipProcessedImfile.class_id)
 LEFT JOIN Label ON chipBackgroundRun.label = Label.label
 WHERE chipBackgroundImfile.chip_bg_id IS NULL
@@ -14 +21 @@
     AND chipProcessedImfile.fault = 0
     AND chipProcessedImfile.quality = 0
+    AND bgsub_chipProcessedImfile.quality = 0
     AND (Label.active OR Label.active IS NULL)
 -- WHERE hook %s

branches/eam_branches/ipp-20130509/ippTools/src/bgtool.c

@@ -774 +774 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "==");
-    pxAddLabelSearchArgs(config, where, "-label", "label", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "==");
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 

branches/eam_branches/ipp-20130509/ippTools/src/chiptool.c

@@ -493 +493 @@
     psFree(where);
 
-    psStringAppend(&query, "\nORDER BY priority DESC, chip_id");
+    psStringAppend(&query, "\nORDER BY priority DESC, chip_id, class_id");
 
     // treat limit == 0 as "no limit"

branches/eam_branches/ipp-20130509/ippTools/src/pstamptool.c

@@ -142 +142 @@
     PXOPT_LOOKUP_STR(label,       config->args, "-set_label",         false, false);
     PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval",false, false);
-    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-set_no_magic", false);
-
-    bool need_magic = ! no_magic;
 
     if (!pstampDataStoreInsert(config->dbh,
@@ -155 +152 @@
             uri,
             pollInterval,
-            need_magic
+            false
         )) {
         psError(PS_ERR_UNKNOWN, false, "database error");

branches/eam_branches/ipp-20130509/ippTools/src/pstamptoolConfig.c

@@ -51 +51 @@
     psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_label",        0, "define datastore label", NULL);
     psMetadataAddS32(adddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 60);
-    psMetadataAddBool(adddatastoreArgs,PS_LIST_TAIL, "-set_no_magic",   0, "set magic not required", true);
 
     // -datastore

branches/eam_branches/ipp-20130509/ppImage/src/Makefile.am

@@ -66 +66 @@
         ppImageAddNoise.c \
         ppImageRandomGaussian.c \
-        ppImageAuxiliaryMask.c
+        ppImageAuxiliaryMask.c \
+        ppImageBurntoolApply.c
 
 CLEANFILES = *~

branches/eam_branches/ipp-20130509/ppImage/src/ppImage.h

@@ -26 +26 @@
     bool doMaskBuild;                   // Build internal mask
     bool doVarianceBuild;               // Build internal variance map
+    bool doApplyBurntool;               // apply burntool correction
     bool doMaskBurntool;                // mask potential burntool trails
     bool doMaskSat;                     // mask saturated pixels
@@ -182 +183 @@
 bool ppImageCheckNoise(pmConfig *config, ppImageOptions *options, pmFPAview *view);
 
-bool ppImageBurntoolMask(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *mask);
+bool ppImageBurntoolMask(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout);
+bool ppImageBurntoolMaskFromTable(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout);
+
+bool ppImageBurntoolApply(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout);
 
 bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, const ppImageOptions *options);

branches/eam_branches/ipp-20130509/ppImage/src/ppImageArguments.c

@@ -117 +117 @@
     pmConfigFileSetsMD (config->arguments, &argc, argv, "LINEARITY", "-linearity", "-linearlist");
 
+    if ((argnum = psArgumentGet(argc, argv, "-burntool"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "BURNTOOL.TABLE", PS_META_REPLACE, "", argv[argnum]);
+        psArgumentRemove(argnum, &argc, argv);
+    }
+
     // chip selection is used to limit chips to be processed
     if ((argnum = psArgumentGet (argc, argv, "-chip"))) {

branches/eam_branches/ipp-20130509/ppImage/src/ppImageAuxiliaryMask.c

@@ -60 +60 @@
 
     if (!options->doAuxMask) {
-        psLogMsg ("ppImage", PS_LOG_INFO, "Auxiliary mask not enabled.");
+        psLogMsg ("ppImage", PS_LOG_DETAIL, "Auxiliary mask not enabled.");
         return true;
     }

branches/eam_branches/ipp-20130509/ppImage/src/ppImageDetrendReadout.c

@@ -56 +56 @@
     }
     done_video_check:
-    
+
     // Masking on the basis of pixel value needs to be done before anything else, so the values are pristine.
     if (options->doMaskBuild) {
@@ -74 +74 @@
       pmMaskBadPixels(input, mask, options->maskValue);
     }
+
+    if (options->doApplyBurntool)
+    {
+        // extern bool ppImageBurntoolApply(pmConfig *, ppImageOptions *, pmFPAview *, pmReadout *);
+        ppImageBurntoolApply(config, options, view, input);
+    }
+    
     if (options->doMaskBurntool) {
-      ppImageBurntoolMask(config,options,view,input);
+        if (options->doApplyBurntool) {
+            // build burntool mask from data input burntool table
+            ppImageBurntoolMaskFromTable(config,options,view,input);
+        } else {
+            // build burntool mask from data in the input image's fits extension
+            ppImageBurntoolMask(config,options,view,input);
+        }
     }
 

branches/eam_branches/ipp-20130509/ppImage/src/ppImageDetrendRecord.c

@@ -76 +76 @@
 
     detrendRecord(options->doNonLin,   detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN",   "Non-linearity table filename");
+
+    detrendRecord(options->doDark & options->useVideoDark, detrend, config, view, "PPIMAGE.VIDEODARK", "DETREND.VIDEODARK", "VideoDark filename");
+    detrendRecord(options->doMask & options->useVideoMask, detrend, config, view, "PPIMAGE.VIDEOMASK", "DETREND.VIDEOMASK", "VideoMASK filename");
     psFree (detrend);
     return true;

branches/eam_branches/ipp-20130509/ppImage/src/ppImageOptions.c

@@ -23 +23 @@
     options->doMaskLow       = false;   // mask low pixels
     options->doMaskBurntool  = false;   // mask potential burntool trails
+    options->doApplyBurntool = false;   // apply burntool correction
     options->doVarianceBuild = false;   // Build internal variance
     options->doMask          = false;   // Mask bad pixels
@@ -231 +232 @@
     options->doMaskLow       = psMetadataLookupBool(NULL, recipe, "MASK.LOW");
     options->doMaskBurntool  = psMetadataLookupBool(NULL, recipe, "MASK.BURNTOOL");
+    options->doApplyBurntool = psMetadataLookupBool(NULL, recipe, "APPLY.BURNTOOL");
     options->doVarianceBuild = psMetadataLookupBool(NULL, recipe, "VARIANCE.BUILD");
     options->doAuxMask       = psMetadataLookupBool(NULL, recipe, "MASK.AUXMASK");

branches/eam_branches/ipp-20130509/ppImage/src/ppImageParseCamera.c

@@ -184 +184 @@
         }
         outCTE->save = true;
+    }
+    if (options->doApplyBurntool) {
+        // If no burntool file was supplied do not fail. (camera_exp.pl does not supply it
+        // for example.
+        if (!psMetadataLookupStr(NULL, config->arguments, "BURNTOOL.TABLE")) {
+            psWarning("BURNTOOL.TABLE not supplied setting doApplyBurntool to false");
+            options->doApplyBurntool = false;
+        }
     }
 

branches/eam_branches/ipp-20130509/psastro/src/psastro.h

@@ -83 +83 @@
 bool              psastroOneChipGrid (pmFPA *fpa, pmChip *chip, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
 bool              psastroOneChipFit (pmFPA *fpa, pmChip *chip, pmReadout *readout, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
-bool              psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source);
+bool              psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source, bool saveExistingMatchedRefs);
 bool              psastroRefstarSubset (pmReadout *readout);
 pmLumFunc        *psastroLuminosityFunction (psArray *stars, pmLumFunc *rawFunc);

branches/eam_branches/ipp-20130509/psastro/src/psastroAnalysis.c

@@ -86 +86 @@
     }
 
-    if (!psastroChooseRefstars (config, refs, "PSASTRO.INPUT")) {
+    bool skipastro = psMetadataLookupBool (&status, config->arguments, "PSASTRO.SKIP.ASTRO");
+
+    if (!psastroChooseRefstars (config, refs, "PSASTRO.INPUT", skipastro)) {
         psError (PSASTRO_ERR_UNKNOWN, false, "failed to select reference data for chips\n");
         psFree(refs);
@@ -108 +110 @@
         mosastro  = psMetadataLookupBool (&status, recipe, "PSASTRO.MOSAIC.MODE");
     }
-    if (!chipastro && !mosastro) {
+
+    if (skipastro) {
+        chipastro = false;
+        mosastro = false;
+        psLogMsg ("psastro", 3, "skip astrometry mode, accepting input astrometry\n");
+    } else if (!chipastro && !mosastro) {
         psLogMsg ("psastro", 3, "no astrometry mode selected, assuming chip astrometry\n");
         chipastro = true;
@@ -144 +151 @@
     }
 
-    if (!psastroZeroPoint (config)) {
-        psError(psErrorCodeLast(), false, "Failed to calculate zero point.");
-        return false;
-    }
+    if (!skipastro) {
+        if (!psastroZeroPoint (config)) {
+            psError(psErrorCodeLast(), false, "Failed to calculate zero point.");
+            return false;
+        }
 
-    if (!psastroAstromGuessCheck (config)) {
-        psError(psErrorCodeLast(), false, "Failed to check astrometry guess.");
-        return false;
+        if (!psastroAstromGuessCheck (config)) {
+            psError(psErrorCodeLast(), false, "Failed to check astrometry guess.");
+            return false;
+        }
     }
 

branches/eam_branches/ipp-20130509/psastro/src/psastroArguments.c

@@ -74 +74 @@
 
     // apply mosastro mode?
+    bool mosastro = false;
     if ((N = psArgumentGet (argc, argv, "-mosastro"))) {
         psArgumentRemove (N, &argc, argv);
@@ -81 +82 @@
         psArgumentRemove (N, &argc, argv);
         psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.MOSAIC.MODE", PS_META_REPLACE, "", true);
+        mosastro = true;
     }
 
     // apply chipastro mode?
+    bool chipastro = false;
     if ((N = psArgumentGet (argc, argv, "-chipastro"))) {
         psArgumentRemove (N, &argc, argv);
@@ -91 +94 @@
         psArgumentRemove (N, &argc, argv);
         psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.CHIP.MODE", PS_META_REPLACE, "", true);
+        chipastro = true;
+    }
+    if ((N = psArgumentGet (argc, argv, "-skipastro"))) {
+        psArgumentRemove (N, &argc, argv);
+        if (mosastro) {
+            psError(PSASTRO_ERR_ARGUMENTS, true, "cannot specify +mosastro with -skipastro");
+            psErrorStackPrint(stderr, "exit");
+            return false;
+        }
+        if (chipastro) {
+            psError(PSASTRO_ERR_ARGUMENTS, true, "cannot specify +chipastro with -skipastro");
+            return false;
+        }
+        psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.SKIP.ASTRO", PS_META_REPLACE, "", true);
     }
 

branches/eam_branches/ipp-20130509/psastro/src/psastroChooseRefstars.c

@@ -20 +20 @@
 }
 
-bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source) {
+bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source, bool saveExistingMatchedRefs) {
 
     bool status;
@@ -55 +55 @@
         // analysis.  however, we are re-doing the astrometry here, so remove
         // that prior set of matched references
-        psMetadataRemoveKey (fpa->analysis, "MATCHED_REFS");
+        if (!saveExistingMatchedRefs) {
+            psMetadataRemoveKey (fpa->analysis, "MATCHED_REFS");
+        }
     }
 

branches/eam_branches/ipp-20130509/psastro/src/psastroExtractAnalysis.c

@@ -44 +44 @@
     }
 
-    if (!psastroChooseRefstars (config, refs, "PSASTRO.EXTRACT.ASTROM")) {
+    if (!psastroChooseRefstars (config, refs, "PSASTRO.EXTRACT.ASTROM", false)) {
         psError (PSASTRO_ERR_UNKNOWN, false, "failed to select reference data for chips\n");
         psFree(refs);

branches/eam_branches/ipp-20130509/pstamp/scripts/pstamp_job_run.pl

@@ -68 +68 @@
         or my_die ("unable to redirect output to $logDest", $job_id, $PS_EXIT_UNKNOWN_ERROR);
 }
-
 
 if (!$dbserver) {

branches/eam_branches/ipp-20130509/pstamp/src/ppstampMakeStamp.c

@@ -814 +814 @@
     psArray *inSources = inDetections->allSources;
 
+    // copy analysis metadata in order to pick up any data stored there from the input file.
+    // This includes detection efficiency analysis and extended source parameters.
+    psMetadataCopy(outReadout->analysis, inReadout->analysis);
+
+    // now replace the sources with a new container
     pmDetections *outDetections = pmDetectionsAlloc();
     psMetadataAddPtr (outReadout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN,
@@ -820 +825 @@
     psArray *outSources = outDetections->allSources = psArrayAllocEmpty( 100 );
 
-    // include sources that are off the stamp by just a little bit
+    // include sources that are off the stamp by just a little bit 
+    // XXX: should I be doing this?
     #define EXPAND_SIZE 16
     psRegion sourceRegion = extractRegion;
@@ -828 +834 @@
     sourceRegion.y1 += EXPAND_SIZE;
 
+    bool adjustCoords = extractRegion.x0 != 0 || extractRegion.y0 != 0;
+
     for (int i = 0; i < inSources->n; i++ ) {
         pmSource *src = inSources->data[i];
@@ -837 +845 @@
             continue;
         }
+        if (adjustCoords) {
+            // adjust the source position for the stamp coordinate system
+            src->peak->x  -= extractRegion.x0;
+            src->peak->xf -= extractRegion.x0;
+
+            src->peak->y  -= extractRegion.y0;
+            src->peak->yf -= extractRegion.y0;
+            if (src->moments) {
+                src->moments->Mx -= extractRegion.x0;
+                src->moments->My -= extractRegion.y0;
+            }
+            if (src->modelPSF) {
+                src->modelPSF->params->data.F32[PM_PAR_XPOS] -= extractRegion.x0;
+                src->modelPSF->params->data.F32[PM_PAR_YPOS] -= extractRegion.y0;
+            }
+
+            if (src->modelFits) {
+                for (int j = 0; j < src->modelFits->n; j++) {
+                    pmModel *model = src->modelFits->data[j];
+                    assert (model != NULL);
+                    model->params->data.F32[PM_PAR_XPOS] -= extractRegion.x0;
+                    model->params->data.F32[PM_PAR_YPOS] -= extractRegion.y0;
+                }
+            }
+        }
+
         psArrayAdd(outSources, 100, src);
     }

branches/eam_branches/ipp-20130509/pswarp/src/pswarpDefineLayout.c

@@ -40 +40 @@
     // chips are not processed unless we have determined they overlap the inputs
     pmFPAExcludeChips (output->fpa);
+    if (bkgModel) {
+        pmFPAExcludeChips (bkgModel->fpa);
+    }
 
     int nInputs = psMetadataLookupS32(&status, config->arguments, "NUM_INPUTS");

branches/eam_branches/ipp-20130509/pswarp/src/pswarpLoadAstrometry.c

@@ -38 +38 @@
         psTrace ("pswarp", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
         if (!chip->process || !chip->file_exists) { continue; }
+        pmChip *targetChip = pmFPAviewThisChip(view, target->fpa); // < Chip in the output
+        if (!targetChip || !targetChip->process || !targetChip->file_exists) {
+            continue; // only load astrometry into output chips which exist!
+        }
         if (!pmFPAfileRead (astrom, view, config)) {
             psError(psErrorCodeLast(), false, "failed READ at CHIP %s", astrom->name);
@@ -109 +113 @@
         if (!chip->process || !chip->file_exists) { continue; }
 
+        pmChip *targetChip = pmFPAviewThisChip(view, target->fpa); // < Chip in the output
+        if (!targetChip || !targetChip->process || !targetChip->file_exists) {
+            continue; // only load astrometry into output chips which exist!
+        }
+
         // we've got the astrom header
         pmHDU *hdu = pmHDUFromChip(chip); ///< HDU for source
@@ -116 +125 @@
             return false;
         }
-
-        // We read from the astrometry source into the target.
-        pmChip *targetChip = pmFPAviewThisChip(view, target->fpa); ///< Chip in the output
-        if (!targetChip) continue; // only load astrometry into output chips which exist!
 
         if (bilevelAstrometry) {

branches/eam_branches/ipp-20130509/pswarp/src/pswarpLoopBackground.c

@@ -36 +36 @@
     }
 
-    // select the input data sources
+    // select the input data background model
     pmFPAfile *output = psMetadataLookupPtr(&status, config->files, "PSWARP.OUTPUT.BKGMODEL");
     if (!output) {

branches/eam_branches/ipp-20130509/tools/runchipimfile.pl

@@ -77 +77 @@
 
 if ($outdir) {
+    $outdir = $ENV{PWD} if $outdir eq '.';
     my $base = basename($path_base);
     $path_base = "$outdir/$base";