Changeset 36192

Timestamp:

Oct 9, 2013, 4:06:09 PM (13 years ago)

Author:

eugene

Message:

merge from trunk

Location:

branches/eam_branches/ipp-20130904

Files:

• Nebulous-Server (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (7 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (3 diffs)
• dbconfig/cam.md (modified) (1 prop)
• dbconfig/changes.txt (modified) (1 prop)
• dvodist/Makefile.in (modified) (2 diffs)
• dvodist/ippdata/index.mkd (modified) (1 diff)
• dvodist/ippdata/ps1.filters (copied) (copied from trunk/dvodist/ippdata/ps1.filters )
• dvodist/ippdata/ps1.filters/apj425122t3_mrt.txt (copied) (copied from trunk/dvodist/ippdata/ps1.filters/apj425122t3_mrt.txt )
• dvodist/ippdata/ps1.filters/index.mkd (copied) (copied from trunk/dvodist/ippdata/ps1.filters/index.mkd )
• ippMonitor (modified) (1 prop)
• ippScripts/scripts/chip_imfile.pl (modified) (1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/lap_science.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/receive_file.pl (modified) (1 diff)
• ippScripts/scripts/warp_skycell.pl (modified) (1 diff, 1 prop)
• ippTasks (modified) (1 prop)
• ippToPsps (modified) (1 prop)
• ippToPsps/jython/dxlayer.py (modified) (2 diffs)
• ippToPsps/jython/ipptopspsdb.py (modified) (5 diffs)
• ippToPsps/jython/stackbatch.py (modified) (2 diffs)
• ippconfig (modified) (1 prop)
• ippconfig/gpc1 (modified) (1 prop)
• ippconfig/gpc1/ppImage.config (modified) (4 diffs)
• ippconfig/gpc1/ppStack.config (modified) (5 diffs)
• ippconfig/recipes/filerules-split.mdc (modified) (1 diff)
• ippconfig/recipes/ppImage.config (modified) (1 diff)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 diff, 1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/psmkreq (modified) (2 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (6 diffs)
• pstamp/scripts/pstampparse.pl (modified) (8 diffs)
• pstamp/test/gpc1/stacksummary.bycoord.all.txt (copied) (copied from trunk/pstamp/test/gpc1/stacksummary.bycoord.all.txt )
• pstamp/test/gpc1/stacksummary.bycoord.txt (modified) (1 diff)
• psvideophot (modified) (1 prop)

branches/eam_branches/ipp-20130904/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -693 +693 @@
                 $out->{data_state} = $out->{state} = $out->{stack_state};
             } else {
-                # XXX: Consider looking up skycal results even if we are
+                $out->{data_state} = $out->{state};
+
+                # XXX: Consider looking up skycal and staticsky results even if we are
                 # not using releasetool
-                $out->{data_state} = $out->{state};
             }
         }
@@ -731 +732 @@
             $out->{mask}   = $ipprc->filename($mask_name,   $mask_base, $class_id) if $mask_name;
             $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name;
-            $out->{cmf}    = $ipprc->filename($cmf_name,    $base, $class_id) if $cmf_name;
-            $out->{psf}    = $ipprc->filename($psf_name,    $base, $class_id) if $psf_name;
             $out->{backmdl}= $ipprc->filename($backmdl_name,$base, $class_id) if $backmdl_name;
+            if ($image->{staticsky_path_base}) {
+                $out->{psf}    = $ipprc->filename("PSPHOT.STACK.PSF.SAVE", $image->{staticsky_path_base}, $image->{stack_id});
+                # if astrom was not set above to the skycal cmf get the sources file from staticsky
+                $out->{cmf}    = $ipprc->filename("PSPHOT.STACK.OUTPUT",   $image->{staticsky_path_base}, $image->{stack_id}) if !$out->{astrom};
+            } else {
+                $out->{psf}    = $ipprc->filename($psf_name,    $base, $class_id) if $psf_name;
+                $out->{cmf}    = $ipprc->filename($cmf_name,    $base, $class_id) if $cmf_name;
+            }
         }
 
@@ -808 +815 @@
                         $command .= " -priority_order";
                     }
-                    $command .= " -surveyName $survey";
+                    $command .= " -surveyName $survey" if $survey;
                 } else {
                     $command = "$stacktool -dbname $imagedb";
@@ -2051 +2058 @@
     # see if we need to start the scs (skycell server)
 
+    my $very_verbose = 0;
+
     if (!$scs_tess_id or ($scs_tess_id ne $tess_id)) {
         if ($scs_tess_id) {
@@ -2076 +2085 @@
 
         # start an scs
+        print "Starting skycell server for $tess_dir_resolved.\n";
 
         # create pipes for communicating with the server
@@ -2111 +2121 @@
             my $command = "$dvoImagesAtCoords -D CATDIR $tess_dir_resolved -coords -";
 
-            print STDERR "SCS: execing $command\n";
+            print STDERR "SCS: execing $command\n" if $very_verbose;
 
             unless(exec $command) {
@@ -2146 +2156 @@
     }
 
-    my $very_verbose = 0;
     print STDERR "  sent coordinates to skycell server\n" if $very_verbose;
 

branches/eam_branches/ipp-20130904/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -37 +37 @@
                     $PSTAMP_USE_IMFILE_ID
                     $PSTAMP_NO_WAIT_FOR_UPDATE
-                    $PSTAMP_REQUEST_UNCENSORED
-                    $PSTAMP_REQUIRE_UNCENSORED
+                    $PSTAMP_SELECT_EXPJPEG
+                    $PSTAMP_SELECT_NUMJPEG
                     $PSTAMP_SUCCESS
                     $PSTAMP_FIRST_ERROR_CODE
@@ -55 +55 @@
                     $PSTAMP_NO_OVERLAP
                     $PSTAMP_NOT_AUTHORIZED
+                    $PSTAMP_NO_VALID_PIXELS
                     );
 our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
@@ -84 +85 @@
 our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
 
-# these bits will be repurposed
-our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
-our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
+our $PSTAMP_SELECT_EXPJPEG     = 0x10000;
+our $PSTAMP_SELECT_NUMJPEG     = 0x20000;
+
+# these bits have been repurposed. They were only exposed to MOPS and IFA and they have adapted.
+#our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
+#our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
 
 # job and result codes

branches/eam_branches/ipp-20130904/dvodist/Makefile.in

@@ -63 +63 @@
 $(TGT-IPPDATA)/photladder.20130107.subset/ubercal-fits/index.php \
 $(TGT-IPPDATA)/photladder.20130107.subset/ubercal-csv/index.php \
+$(TGT-IPPDATA)/ps1.filters/index.php \
 $(TGT-IPPDATA)/catdir.syn.test/index.php \
 $(TGT-IPPDATA)/catdir.syn.test/n3000/index.php \
@@ -73 +74 @@
 $(TGT-IPPDATA)/ipp.css \
 $(TGT-IPPDATA)/PScolorlogo2.jpg \
+$(TGT-IPPDATA)/ps1.filters/apj425122t3_mrt.txt \
 $(TGT-IPPDATA)/docs/photladder-1206.20130301.pdf \
 $(TGT-IPPDATA)/docs/2mass.compare/dH.allsky.20120528.v3.png \

branches/eam_branches/ipp-20130904/dvodist/ippdata/index.mkd

@@ -45 +45 @@
 @ PS1 Photladder, 12.01       : photladder.20130107.subset : dir   : dir1  : Photometric Reference Ladder, Release 12.01
 @ Synthetic Photometry DVO    : catdir.syn.test            : dir   : dir2  : Demonstration DVO Database
+@ PS1 Filter Bandpasses       : ps1.filters                : dir   : dir1  : PS1 Filter Bandpass Tables

branches/eam_branches/ipp-20130904/ippScripts/scripts/receive_file.pl

@@ -239 +239 @@
     }
 
-    if ($stage eq 'sky' or $stage = 'skycal') {
+    if ($stage eq 'sky' or $stage eq 'skycal') {
         # the dbinfo file for staticskyRun and skycalRun only have one component and they don't contain
         # skycell_id which is the way components are listed in the dirinfo file.
         my @ids = keys %$components;
-        &my_die( "unexpected number of components scalar @ids found in staticsky dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if scalar @ids != 1;
+        my $nComponents = scalar @ids;
+        &my_die( "unexpected number of components $nComponents found in $stage dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if $nComponents  != 1;
         $current_component = $ids[0];
     }

branches/eam_branches/ipp-20130904/ippScripts/scripts/warp_skycell.pl

@@ -388 +388 @@
             $quality = 8006; # bad data.
         }
-#         if (!$quality) {
-#             check_output($outputImage, 0);
-#             check_output($outputMask, 0);
-#             check_output($outputWeight, 0);
-#             check_output($outputSources, 1);
-#             check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF')  ;
-#         }
         print "Quality: $quality\n";
+        if (!$quality) {
+            check_output($outputImage, 0);
+            check_output($outputMask, 0);
+            check_output($outputWeight, 0);
+            check_output($outputSources, 0);
+            check_output($outputPSF, 0) if metadataLookupBool($recipe, 'PSF')  ;
+        }
 
         # $run_state eq 'update'

branches/eam_branches/ipp-20130904/ippToPsps/jython/dxlayer.py

@@ -25 +25 @@
     '''
     def deleteBatch(self, batchID):
+        #HAF This is really lazy to do this way, but for the time, will check both dx layers and try the first, if no go try the second.
 
         batch = Batch.getNameFromID(batchID)
@@ -46 +47 @@
         p.stdout.close()
 
-        if ret: self.logger.infoPair("Deleted batch from DXLayer", batch) 
-        else: self.logger.errorPair("Unable to delete batch from DXLayer", batch)
+        if ret: self.logger.infoPair("Deleted batch from DXLayer 1", batch) 
+        else: self.logger.errorPair("Unable to delete batch from DXLayer 1" , batch)
+
+
+       
+
+        tempFile = tempfile.NamedTemporaryFile(mode='w+b')
+
+        p = Popen("deleteFromDXLayer2.pl -b " + batch + " -f " + tempFile.name, 
+                shell=True, 
+                stdout=PIPE)
+        p.wait()
+       
+        if p.returncode != 0:
+           self.logger.debugPair("DXLayer webservice", "failed")
+           return 0
+        else:
+           self.logger.debugPair("DXLayer webservice", "successful")
+
+        ret = self.decodeDXLayerResponse(tempFile.name)
+
+        tempFile.close()
+        p.stdout.close()
+
+        if ret: self.logger.infoPair("Deleted batch from DXLayer 2", batch) 
+        else: self.logger.errorPair("Unable to delete batch from DXLayer 2" , batch)
+
+
+
 
         return ret
+
 
     '''

branches/eam_branches/ipp-20130904/ippToPsps/jython/ipptopspsdb.py

@@ -39 +39 @@
                AND batch_type = '" + batchType + "' \
                AND dvo_db = '" + self.skychunk.dvoLabel + "' \
+               AND comment = '" + self.skychunk.name + "' \
+               AND datastore_product = '" + self.skychunk.datastoreProduct + "' \
                AND merged = 1 \
                AND deleted_local = 0 \
@@ -65 +67 @@
                AND batch_type = '" + batchType + "' \
                AND dvo_db = '" + self.skychunk.dvoLabel + "' \
+               AND comment = '" + self.skychunk.name + "' \
+               AND datastore_product = '" + self.skychunk.datastoreProduct + "' \
                AND purged = 1 \
                AND deleted_local = 0 \
@@ -92 +96 @@
                AND batch_type = '" + batchType + "' \
                AND dvo_db = '" + self.skychunk.dvoLabel + "' \
+               AND comment = '" + self.skychunk.name + "' \
+               AND datastore_product = '" + self.skychunk.datastoreProduct + "' \
                AND purged = 1 \
                AND deleted_datastore = 0 \
@@ -119 +125 @@
                AND batch_type = '" + batchType + "' \
                AND dvo_db = '" + self.skychunk.dvoLabel + "' \
-               AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
+               AND comment = '" + self.skychunk.name + "' \
+               AND datastore_product = '" + self.skychunk.datastoreProduct + "' \
+               AND (merge_worthy = 1) \
                AND deleted_datastore = 0 AND block_delete_datastore = 0"
 
@@ -144 +152 @@
                AND batch_type = '" + batchType + "' \
                AND dvo_db = '" + self.skychunk.dvoLabel + "' \
-               AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
+               AND (merge_worthy = 1) \
                AND deleted_dxlayer = 0 and block_delete_dxlayer = 0"
 

branches/eam_branches/ipp-20130904/ippToPsps/jython/stackbatch.py

@@ -645 +645 @@
         ,petRadiusErr=b.PETRO_RADIUS_ERR \
         ,petFlux=POW(10.0, -0.4 * b.PETRO_MAG) /" + str(self.expTime) + " \
-        ,petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736  \
+        ,petFluxErr=-999  \
         ,petR50=b.PETRO_RADIUS_50 \
         ,petR50Err=b.PETRO_RADIUS_50_ERR \
@@ -653 +653 @@
         WHERE a.ippDetectID=b.IPP_IDET"
         self.scratchDb.execute(sql)
+
+        self.logger.infoPair("Adding", "petFluxErr for non nulls")
+        sql = "UPDATE StackApFlx as a, SkyChip_xsrc AS b SET \
+        petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736  \
+        WHERE a.ippDetectID=b.IPP_IDET \
+        AND (b.PETRO_MAG_ERR is not NULL AND b.PETRO_MAG is not NULL AND PETRO_MAG_ERR < 1000.0)"
+        # added this because when the petromagerr is crazy large (10^29 it out of ranges the flux err calc)
+        self.logger.infoPair("sql", sql)
+        self.scratchDb.execute(sql)
+
+
 
         self.logger.infoPair("adding"," PrimaryF to stackApFlx")

branches/eam_branches/ipp-20130904/ippconfig/gpc1/ppImage.config

@@ -34 +34 @@
 # aditional auxiliary mask applied to corners of all chips in video cells.
 # Is only applied if MASK.AUXMASK and USE.VIDEO.MASK are both true
-AUX.VIDEO.MASK  STR     neb://any/gpc1/auxmask/mpg.20130506/mask_video_cell_corners.fits
+AUX.VIDEO.MASK  STR     neb://any/gpc1/auxmask/mpg.20130930/mask_video_cell_corners.fits
 
 
@@ -189 +189 @@
 END
 
+CHIP_NOCOMPRESS METADATA
+  BASE.FITS          BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS     BOOL    FALSE           # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS          BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS     BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
+  OVERSCAN           BOOL    TRUE            # Overscan subtraction
+  NONLIN             BOOL    TRUE            # apply non-linearity correction 
+  BIAS               BOOL    FALSE           # Bias subtraction
+  DARK               BOOL    TRUE            # Dark subtraction
+  NOISEMAP           BOOL    TRUE            
+  REMNANCE           BOOL    FALSE           # Remnance masking
+  SHUTTER            BOOL    FALSE           # Shutter correction
+  FLAT               BOOL    TRUE            # Flat-field normalisation
+  MASK               BOOL    TRUE            # Mask bad pixels
+  MASK.BUILD         BOOL    TRUE            # Build internal mask?
+  MASK.BURNTOOL      BOOL    TRUE            # Mask potential burntool trails
+  MASK.AUXMASK       BOOL    FALSE           # augment mask with more restrictive auxiliary mask
+  PATTERN.ROW        BOOL    TRUE            # Row pattern correction
+  PATTERN.CELL       BOOL    FALSE           # Cell pattern correction
+  PATTERN.CONTINUITY BOOL    TRUE            # Cell pattern correction using edge continuity
+  FRINGE             BOOL    TRUE            # Fringe subtraction
+  BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS          BOOL    TRUE            # Save 2nd binned chip image?
+  PHOTOM             BOOL    TRUE            # Source identification and photometry
+  ASTROM.CHIP        BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC      BOOL    FALSE           # Astrometry for mosaic?
+  BACKGROUND         BOOL    TRUE            # Subtract background?
+  CROSSTALK.MEASURE  BOOL    FALSE           # Subtract model background?
+  NO.COMPRESS        BOOL    TRUE            # do not compress images
+END
+
 # Chip processing for testing pattern correction
 CHIP_PATTERNTEST METADATA
@@ -1259 +1292 @@
   CROSSTALK.MEASURE  BOOL    FALSE           # Subtract model background?
   NO.COMPRESS        BOOL    TRUE            # turn off image compression
+    DETREND.CONSTRAINTS  METADATA
+      BIAS METADATA
+      END
+      DARK METADATA
+        EXPTIME STR FPA.EXPOSURE
+      END
+      FLAT METADATA
+        FILTER  STR FPA.FILTERID
+      END
+      FLATCORR METADATA
+        FILTER   STR FPA.FILTERID
+      END
+      FRINGE METADATA
+        FILTER   STR FPA.FILTERID
+      END
+      SHUTTER METADATA
+      END
+      MASK METADATA
+      END       
+      AUXMASK METADATA
+        DETTYPE STR AUXMASK_STS
+      END
+      ASTROM METADATA
+      END       
+      NOISEMAP METADATA
+      END       
+      LINEARITY METADATA
+      END
+    END
+END
+
+# Standard chip processing + auxiliary mask with heavy masking
+CHIP_AUXMASK_HEAVY METADATA
+  BASE.FITS          BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS     BOOL    FALSE           # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS          BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS     BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
+  OVERSCAN           BOOL    TRUE            # Overscan subtraction
+  NONLIN             BOOL    TRUE            # apply non-linearity correction 
+  BIAS               BOOL    FALSE           # Bias subtraction
+  DARK               BOOL    TRUE            # Dark subtraction
+  NOISEMAP           BOOL    TRUE            
+  REMNANCE           BOOL    FALSE           # Remnance masking
+  SHUTTER            BOOL    FALSE           # Shutter correction
+  FLAT               BOOL    TRUE            # Flat-field normalisation
+  MASK               BOOL    TRUE            # Mask bad pixels
+  MASK.BUILD         BOOL    TRUE            # Build internal mask?
+  MASK.BURNTOOL      BOOL    TRUE            # Mask potential burntool trails
+  MASK.AUXMASK       BOOL    TRUE            # augment mask with more restrictive auxiliary mask
+  PATTERN.ROW        BOOL    TRUE            # Row pattern correction
+  PATTERN.CELL       BOOL    FALSE           # Cell pattern correction
+  PATTERN.CONTINUITY BOOL    TRUE            # Cell pattern correction using edge continuity
+  FRINGE             BOOL    TRUE            # Fringe subtraction
+  BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS          BOOL    TRUE            # Save 2nd binned chip image?
+  PHOTOM             BOOL    TRUE            # Source identification and photometry
+  ASTROM.CHIP        BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC      BOOL    FALSE           # Astrometry for mosaic?
+  BACKGROUND         BOOL    TRUE            # Subtract background?
+  CROSSTALK.MEASURE  BOOL    FALSE           # Subtract model background?
+  NO.COMPRESS        BOOL    TRUE            # turn off image compression
+    DETREND.CONSTRAINTS  METADATA
+      BIAS METADATA
+      END
+      DARK METADATA
+        EXPTIME STR FPA.EXPOSURE
+      END
+      FLAT METADATA
+        FILTER  STR FPA.FILTERID
+      END
+      FLATCORR METADATA
+        FILTER   STR FPA.FILTERID
+      END
+      FRINGE METADATA
+        FILTER   STR FPA.FILTERID
+      END
+      SHUTTER METADATA
+      END
+      MASK METADATA
+      END       
+      AUXMASK METADATA
+        DETTYPE STR AUXMASK_HEAVY
+      END
+      ASTROM METADATA
+      END       
+      NOISEMAP METADATA
+      END       
+      LINEARITY METADATA
+      END
+    END
 END
 
@@ -1359 +1484 @@
   CROSSTALK.MEASURE  BOOL    FALSE           # Subtract model background?
   NO.COMPRESS        BOOL    TRUE            # turn off image compression
+    DETREND.CONSTRAINTS  METADATA
+      BIAS METADATA
+      END
+      DARK METADATA
+        EXPTIME STR FPA.EXPOSURE
+      END
+      FLAT METADATA
+        FILTER  STR FPA.FILTERID
+      END
+      FLATCORR METADATA
+        FILTER   STR FPA.FILTERID
+      END
+      FRINGE METADATA
+        FILTER   STR FPA.FILTERID
+      END
+      SHUTTER METADATA
+      END
+      MASK METADATA
+      END       
+      AUXMASK METADATA
+        DETTYPE STR AUXMASK_HEAVY
+      END
+      ASTROM METADATA
+      END       
+      NOISEMAP METADATA
+      END       
+      LINEARITY METADATA
+      END
+    END
 END
 

branches/eam_branches/ipp-20130904/ippconfig/gpc1/ppStack.config

@@ -125 +125 @@
 END
 
-STACK_NIGHTLY_1DG  METADATA                 # Typically MD 8 input nightly stacks and using 1DG convolution
-    OUTPUT.NOCOMP      BOOL  FALSE
-    OUTPUT.LOGFLUX     BOOL  FALSE
-    PSF.TARGET.AS.MAX  BOOL  TRUE
-    PSF.TARGET.AS.MAX.EPSILON F32 0.1
+STACK_NIGHTLY_1DG  METADATA                 # Revised MD 8 input nightly stacks and using 1DG convolution
+    OUTPUT.NOCOMP             BOOL  FALSE
+    OUTPUT.LOGFLUX            BOOL  FALSE
+    PSF.INPUT.CLIP.SIMPLE     BOOL  True
+    PSF.INPUT.MAX             F32   12.0    # what FWHM does diffim become mostly useless, if any?
+    PSF.INPUT.CLIP.NSIGMA     F32   1000.0  # want to use up to MAX, but sig could possibly be small?
+    MATCH.REJ                 F32   4.0     # need to tune?
+    SAFE                      BOOL  FALSE
+    PSF.TARGET.AS.MAX         BOOL  TRUE
+    PSF.TARGET.AS.MAX.EPSILON F32   0.1
 END
 
@@ -138 +143 @@
     STACK.TYPE              STR   DEEP_STACK  ## needed?
     PSF.INPUT.CLIP.SIMPLE   BOOL  True
-    PSF.INPUT.MAX           F32   6.0   # never really want >6 pixels in refstack
+    PSF.INPUT.MAX           F32   6.0   # never really want >6 pixels in refstack, even on edges 
     PSF.INPUT.CLIP.NSIGMA   F32   1.0   # sample typically on rising side of distribution (set even smaller?)
     PSF.INPUT.THRESH        F32   NAN
     PSF.INPUT.ASYMMETRY     F32   NAN
-    MATCH.REJ               F32   4.0 # need ot tune?
+    MATCH.REJ               F32   4.0   # need ot tune?
     SAFE                    BOOL  FALSE
     PSF.TARGET.AS.MAX       BOOL  TRUE      # Set the target PSF FWHM as the maximum of accepted input FWHM values.
@@ -154 +159 @@
     STACK.TYPE              STR   DEEP_STACK
     PSF.INPUT.CLIP.SIMPLE   BOOL  True
-    PSF.INPUT.MAX           F32   10.0
+    PSF.INPUT.MAX           F32   10.0    #  
     PSF.INPUT.CLIP.NSIGMA   F32   100.0
     PSF.INPUT.THRESH        F32   NAN
     PSF.INPUT.ASYMMETRY     F32   NAN
-    MATCH.REJ               F32   4.0 # need ot tune?
+    MATCH.REJ               F32   4.0     # need ot tune?
     SAFE                    BOOL  FALSE
     PSF.TARGET.AS.MAX  BOOL  TRUE
@@ -164 +169 @@
 END
 
-STACK_ALLDEEP_1DG      METADATA     # no cuts, everything in changes after ppStack mods 5/03/13
+STACK_ALLDEEP_1DG      METADATA     # no real cuts, for changes after ppStack mods 5/03/13
     OUTPUT.NOCOMP           BOOL  TRUE
     OUTPUT.LOGFLUX          BOOL  FALSE
@@ -170 +175 @@
     STACK.TYPE              STR   DEEP_STACK
     PSF.INPUT.CLIP.SIMPLE   BOOL  True
-    PSF.INPUT.MAX           F32   100.0
+    PSF.INPUT.MAX           F32   14.0    # greater than 14 is more often junk
     PSF.INPUT.CLIP.NSIGMA   F32   100.0
     PSF.INPUT.THRESH        F32   NAN
     PSF.INPUT.ASYMMETRY     F32   NAN
-    MATCH.REJ               F32   4.0 # need ot tune?
+    MATCH.REJ               F32   4.0     # need ot tune?
     SAFE                    BOOL  FALSE
     PSF.TARGET.AS.MAX  BOOL  TRUE

branches/eam_branches/ipp-20130904/ippconfig/recipes/filerules-split.mdc

@@ -38 +38 @@
 PPIMAGE.INPUT.MASK        INPUT    @FILES        CHIP       MASK
 PPIMAGE.VIDEOMASK         INPUT    @DETDB        CHIP       MASK
+PPIMAGE.AUXMASK           INPUT    @DETDB        CHIP       MASK
 PPIMAGE.INPUT.VARIANCE    INPUT    @FILES        CHIP       VARIANCE
 PPIMAGE.INPUT.PSF         INPUT    @FILES        CHIP       PSF

branches/eam_branches/ipp-20130904/ippconfig/recipes/ppImage.config

@@ -194 +194 @@
   MASK.STATS         BOOL    TRUE            # Calculate Mask statistics.
 END
-
 # Standard chip processing
+CHIP_NOCOMPRESS METADATA
+  BASE.FITS          BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS     BOOL    FALSE           # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS          BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS     BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
+  OVERSCAN           BOOL    TRUE            # Overscan subtraction
+  NONLIN             BOOL    FALSE           # Non-linearity correction; not implemented
+  BIAS               BOOL    TRUE            # Bias subtraction
+  DARK               BOOL    TRUE            # Dark subtraction
+  SHUTTER            BOOL    FALSE           # Shutter correction
+  FLAT               BOOL    TRUE            # Flat-field normalisation
+  MASK               BOOL    TRUE            # Mask bad pixels
+  FRINGE             BOOL    TRUE            # Fringe subtraction
+  BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS          BOOL    TRUE            # Save 2nd binned chip image?
+  PHOTOM             BOOL    TRUE            # Source identification and photometry
+  ASTROM.CHIP        BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC      BOOL    FALSE           # Astrometry for mosaic?
+  BACKGROUND         BOOL    TRUE            # Subtract background?
+  MASK.STATS         BOOL    TRUE            # Calculate Mask statistics.
+  NO.COMPRESS        BOOL    TRUE
+END
+
+# Standard chip processing with auxillary masking
 CHIP_AUXMASK    METADATA
+  BASE.FITS          BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS     BOOL    FALSE           # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS          BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS     BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
+  OVERSCAN           BOOL    TRUE            # Overscan subtraction
+  NONLIN             BOOL    FALSE           # Non-linearity correction; not implemented
+  BIAS               BOOL    TRUE            # Bias subtraction
+  DARK               BOOL    TRUE            # Dark subtraction
+  SHUTTER            BOOL    FALSE           # Shutter correction
+  FLAT               BOOL    TRUE            # Flat-field normalisation
+  MASK               BOOL    TRUE            # Mask bad pixels
+  FRINGE             BOOL    TRUE            # Fringe subtraction
+  BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS          BOOL    TRUE            # Save 2nd binned chip image?
+  PHOTOM             BOOL    TRUE            # Source identification and photometry
+  ASTROM.CHIP        BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC      BOOL    FALSE           # Astrometry for mosaic?
+  BACKGROUND         BOOL    TRUE            # Subtract background?
+  MASK.STATS         BOOL    TRUE            # Calculate Mask statistics.
+END
+
+# Standard chip processing with heavy auxillary masking (same as CHIP_AUXMASK here, but different in gpc1)
+CHIP_AUXMASK_HEAVY      METADATA
   BASE.FITS          BOOL    FALSE           # Save base detrended image?
   BASE.MASK.FITS     BOOL    FALSE           # Save base detrended image?

branches/eam_branches/ipp-20130904/ippconfig/recipes/reductionClasses.mdc

@@ -1129 +1129 @@
    CHIP_PSPHOT STR CHIP
    WARP_PSWARP STR WARP
+   STACK_PPSTACK STR STACK
+   STACK_PPSUB STR STACK
+   STACK_PSPHOT STR STACK
+   DIFF_PPSUB  STR DIFF
+   DIFF_PSPHOT STR DIFF
+   JPEG_BIN1   STR PPIMAGE_J1
+   JPEG_BIN2   STR PPIMAGE_J2
+   FAKEPHOT    STR FAKEPHOT
+   ADDSTAR     STR ADDSTAR
+   PSASTRO     STR STS_DATASET
+   STACKPHOT   STR STACKPHOT
+END
+
+STS_DATASET_NOCOMPRESS METADATA
+   CHIP_PPIMAGE STR CHIP_NOCOMPRESS
+   CHIP_PSPHOT STR CHIP
+   WARP_PSWARP STR WARP_NOCOMPRESS
    STACK_PPSTACK   STR STACK
    STACK_PPSUB STR STACK

branches/eam_branches/ipp-20130904/pstamp/scripts/psmkreq

@@ -177 +177 @@
 $id = 0 if !$id;
 
-unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') {
+unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack'
+    or $stage eq 'stack_summary') {
     die "$stage is not a valid value for stage\n";
 }
@@ -208 +209 @@
         $option_mask |= $PSTAMP_SELECT_PSF      if $psf;
         $option_mask |= $PSTAMP_SELECT_BACKMDL  if $backmdl;
+        $option_mask |= $PSTAMP_SELECT_UNCOMPRESSED   if $uncompressed;
         $option_mask |= $PSTAMP_SELECT_UNCONV   if $unconvolved;
         $option_mask |= $PSTAMP_USE_IMFILE_ID   if $use_imfile_id;

branches/eam_branches/ipp-20130904/pstamp/scripts/pstamp_checkdependent.pl

@@ -304 +304 @@
             }
             $queued_update = 1;
+        } elsif ($chip->{state} eq 'cleaned' and $chip->{data_state} eq 'update') {
+            # we've had a number of runs in this limbo state
+
+            my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
+            $command .= " -set_label $rlabel" if $rlabel;
+
+            if (!$no_update) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    my_die("failed to change ${stage}Run $stage_id $component from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+                }
+            } else {
+                print "skipping $command\n";
+            }
+            $queued_update = 1;
         } elsif ($chip->{fault}) {
             $fault_count++;
@@ -460 +476 @@
             print "skipping $command\n";
         }
+    } elsif ($chips_ready and $skycell->{data_state} eq 'update' and $skycell->{state} ne 'update') {
+        my $command = "$warptool -updaterun -warp_id $warp_id -set_state update";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to change state of ${stage}Run $stage_id to update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+
     } elsif (scalar @chipsToUpdate > 0) {
         my $fault = check_states_chip($chip_id, \@chipsToUpdate, $rlabel, $need_magic);

branches/eam_branches/ipp-20130904/pstamp/scripts/pstamp_job_run.pl

@@ -236 +236 @@
                            $PSTAMP_SELECT_JPEG     => 'jpg',
                            $PSTAMP_SELECT_EXP      => 'exp.fits',
-                           $PSTAMP_SELECT_NUM      => 'num.fits');
+                           $PSTAMP_SELECT_NUM      => 'num.fits',
+                           $PSTAMP_SELECT_EXPJPEG  => 'exp.jpg',
+                           $PSTAMP_SELECT_NUMJPEG  => 'num.jpg');
 
         my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE 
-            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
+            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES
+            | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM 
+            | $PSTAMP_SELECT_EXPJPEG | $PSTAMP_SELECT_NUMJPEG);
+
 
         foreach my $key (keys (%extensions)) {
@@ -261 +266 @@
         close $F;
         $jobStatus = $PS_EXIT_SUCCESS;
-    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
-        $jobStatus = $PSTAMP_NO_OVERLAP;
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP || $exitStatus == $PSTAMP_NO_VALID_PIXELS) {
+        $jobStatus = $exitStatus;
     } else {
         my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
@@ -382 +387 @@
         my $backmdl_file = $params->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $filter = $params->{filter};
+        $filter = ' ' if !$filter;
+        $filter = substr $filter, 0, 1;
         my $cmf_file;
 #        if ($stage ne 'chip') {
@@ -394 +402 @@
             if (!$therest or !$rownum or !$jobnum);
 
-        my $prefix = "${rownum}_${jobnum}_";
+        # XXX: Here we are assuming the form of the output file names
+        # if we change this in pstampparse we'll need to remember ....
+        # (the last time I forgot)
+        my $prefix = "${rownum}_${jobnum}_${filter}_";
 
         if ($cmf_file) {
@@ -573 +584 @@
     $params->{exp}    = $path_base . ".exp.b1.fits";
     $params->{num}    = $path_base . ".num.b1.fits";
+    $params->{expjpeg} = $path_base . ".exp.0.b1.jpeg";
+    $params->{numjpeg} = $path_base . ".num.0.b1.jpeg";
 }
 
@@ -632 +645 @@
         if (!myCopy("$outputBase.num.fits", $params->{num}, 'num', 0)) {
             $options &= ~$PSTAMP_SELECT_NUM;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXPJPEG) {
+        if (!myCopy("$outputBase.exp.jpg", $params->{expjpeg}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXPJPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUMJPEG) {
+        if (!myCopy("$outputBase.num.jpg", $params->{numjpeg}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUMJPEG;
         }
     }

branches/eam_branches/ipp-20130904/pstamp/scripts/pstampparse.pl

@@ -31 +31 @@
 my $no_update;
 my $dest_requires_magic;
+my $dump_params = 0;
 
 # set this to true to disable update processing
@@ -48 +49 @@
     'save-temps'=>  \$save_temps,
     'no-update' =>  \$no_update,
+    'dump-params' => \$dump_params,
 );
 
@@ -238 +240 @@
 
 
+# if label is for one of the high priority channels, watch for big requests
+my $watch_for_big_requests = (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/));
+
+my $big_limit = 100;    # XXX: this should be in a configuration file somewhere not hard coded
+
+if ($watch_for_big_requests and $nRows > $big_limit) {
+    $label = change_to_lower_priority_label($label);
+    $watch_for_big_requests = 0;
+}
 
 my $num_jobs = 0;
-my $imageList;
-my $stage;
-my $big_limit = 100;    # XXX: this should be in a configuration file some where
+
 foreach my $row (@$rows) {
-
-
-    if (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/) and ($nRows > $big_limit or $num_jobs > $big_limit)
-        and $req_id and !$no_update) {
-
-        # This is a "big" request and it came from one of the "high priority" channels
-        # and doesn't have a specific label assigned.
-        # Change the label to a value that its jobs run with lower priority.
-
-        my $old_label = $label;
-        $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
-        print "\nChanging label for big $old_label request to $label\n";
-
-        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-        }
-    }
 
     # validate the paramaters
@@ -276 +265 @@
 
     $num_jobs += processRow($action, $row);
+
+    # see whether number of jobs limit for high priority request was exceeded
+    if ($watch_for_big_requests and $num_jobs > $big_limit) {
+        $label = change_to_lower_priority_label($label);
+        $watch_for_big_requests = 0;
+    }
 }
 
@@ -286 +281 @@
 
 exit 0;
+
+# end of main function
+
+sub change_to_lower_priority_label {
+    my $label = shift;
+    my $old_label = $label;
+    $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
+    print "\nChanging label for big $old_label request to $label\n";
+
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    return $label;
+}
 
 sub checkRow {
@@ -494 +506 @@
     print "\nCalling locate_images_for_row for row: $rownum\n";
 
-    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+    my $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
 
     my $dtime_locate = gettimeofday() - $start_locate;
@@ -586 +598 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
+    my $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -777 +789 @@
             print "$image->{image}\n";
             ++$firstRow->{job_num};
+            if ($dump_params) {
+                my $rownum = $firstRow->{ROWNUM};
+                my $jobnum = $firstRow->{job_num};
+                my $filter = substr $image->{filter}, 0, 1;
+                my $output_base = "${rownum}_${jobnum}_${filter}";
+                write_params($output_base, $image);
+            }
         }
     } elsif ($job_type eq "get_image") {

branches/eam_branches/ipp-20130904/pstamp/test/gpc1/stacksummary.bycoord.txt

@@ -4 +4 @@
 CHANGEME 2 PROCESS null
 #  ROWNUM CENTER_X CENTER_Y WIDTH HEIGHT COORD_MASK JOB_TYPE OPTION_MASK PROJECT SURVEY_NAME IPP_RELEASE REQ_TYPE IMG_TYPE ID TESS_ID COMPONENT DATA_GROUP REQFILT MJD_MIN MJD_MAX RUN_TYPE FWHM_MIN FWHM_MAX | COMMENT
-1     359.767   -30.2009     100    100   2      stamp      455         gpc1      3PI         null     bycoord   stack_summary  null RINGS.V3 null   LAP.ThreePI.20130717%      r        0      0        null      0        0    | 
+1     359.767   -30.2009     100    100   2  stamp      196673        gpc1      3PI         null     bycoord   stack_summary  null RINGS.V3 null   LAP.ThreePI.20130717%      r        0      0        null      0        0    |