Changeset 35413 for branches/eam_branches/ipp-20130307

Timestamp:

Apr 19, 2013, 3:42:02 PM (13 years ago)

Author:

eugene

Message:

merged from trunk

Location:

branches/eam_branches/ipp-20130307

Files:

• . (modified) (1 prop)
• DataStore/lib/DataStore/Utils.pm (modified) (1 diff)
• DataStoreServer/scripts/dsreg (modified) (1 diff)
• Nebulous (modified) (1 prop)
• Nebulous-Server (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (37 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (4 diffs)
• dbconfig/cam.md (modified) (1 prop)
• dbconfig/changes.txt (modified) (1 prop)
• ippMonitor (modified) (1 prop)
• ippMonitor/raw/czartool_getplot.php (modified) (1 diff)
• ippMonitor/raw/heathers_plot.php (modified) (2 diffs)
• ippTasks (modified) (1 prop)
• ippTasks/release.pro (modified) (1 diff)
• ippTasks/survey.pro (modified) (8 diffs)
• ippToPsps (modified) (1 prop)
• ippconfig (modified) (1 prop)
• ippconfig/dvo.photcodes (modified) (2 diffs)
• ippconfig/gpc1/ppStack.config (modified) (5 diffs)
• ippconfig/gpc1/psphot.config (modified) (1 prop)
• ippconfig/isp/dvo.config (modified) (1 diff)
• ippconfig/isp/ppImage.config (modified) (1 diff)
• ippconfig/recipes/filerules-mef.mdc (modified) (1 diff)
• ippconfig/recipes/filerules-simple.mdc (modified) (1 diff)
• ippconfig/recipes/filerules-split.mdc (modified) (1 diff)
• ippconfig/recipes/ppStack.config (modified) (1 diff)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/psphot.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)
• ppImage/src (modified) (1 prop)
• ppImage/src/ppImageParseCamera.c (modified) (1 diff)
• ppStack/src/ppStackMatch.c (modified) (7 diffs)
• ppStack/src/ppStackPrepare.c (modified) (6 diffs)
• ppStack/src/ppStackSources.c (modified) (11 diffs)
• ppStack/src/ppStackTarget.c (modified) (2 diffs)
• psModules (modified) (1 prop)
• psModules/src/objects/pmSourceIO.c (modified) (2 diffs)
• psModules/src/objects/pmSourceMatch.c (modified) (3 diffs)
• psModules/src/objects/pmSourceMatch.h (modified) (1 diff)
• psconfig (modified) (1 prop)
• psphot (modified) (1 prop)
• psphot/src (modified) (1 prop)
• psphot/src/psphotStackImageLoop.c (modified) (1 prop)
• psvideophot (modified) (1 prop)
• pswarp/doc/notes.20130406.txt (modified) (1 diff)
• pswarp/src/Makefile.am (modified) (1 diff)
• pswarp/src/pswarp.c (modified) (1 diff)
• pswarp/src/pswarpDefine.c (modified) (1 diff)
• pswarp/src/pswarpDefineSkycell.c (modified) (2 diffs)
• pswarp/src/pswarpLoop.c (modified) (2 diffs)
• pswarp/src/pswarpParseCamera.c (modified) (4 diffs)

branches/eam_branches/ipp-20130307/DataStore/lib/DataStore/Utils.pm

@@ -42 +42 @@
 $BYTE_FIELD = qr/^\d+$/;
 $MD5_FIELD = qr/^[0-9a-f]{32}$/;
-%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits IPP-MOPS notset );
+%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits IPP-MOPS IPP-PSPS ipp-psps notset );
 %KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS IPP-DIST table);
 %KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc dqresults);

branches/eam_branches/ipp-20130307/DataStoreServer/scripts/dsreg

@@ -236 +236 @@
     $dbh->disconnect();
 
-    if ($remove) {
+    if ($remove and -e $fileset_dir) {
         my $rc;
-        if (($rc = system "rm -r $fileset_dir")) {
+        if (($rc = system "rm -rf $fileset_dir")) {
             &my_die("failed to remove $fileset_dir error code: $rc", $PS_EXIT_UNKNOWN_ERROR);
         }

branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -15 +15 @@
 our @EXPORT_OK = qw( 
                     locate_images
+                    locate_images_for_row
                     resolve_project
                     getCamRunByCamID
@@ -46 +47 @@
 my $last_project = "";
 
-my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool);
+my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool, $releasetool);
 my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump);
 
@@ -56 +57 @@
     my $rowList  = shift;   # required
     my $req_type = shift;   # required
-    my $img_type = shift;   # required
+    my $stage = shift;   # required
     my $id       = shift;   # required unless req_type eq bycoord or byskycell
     my $tess_id  = shift;
@@ -78 +79 @@
            ($req_type ne "byskycell");
 
-    $default_data_groups = load_data_groups($verbose);
+    if (!$data_group and !$default_data_groups) {
+        $default_data_groups = load_data_groups($verbose);
+    }
 
     my $dateobs_begin;
@@ -104 +107 @@
         my $x = $row->{CENTER_X};
         my $y = $row->{CENTER_Y};
-        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
         return $results;
     }
 
-    if (($req_type eq "byid") and ($img_type eq "diff")) {
+    if (($req_type eq "byid") and ($stage eq "diff")) {
         # lookups of all of the information for diff images requires a two level lookup to
         # get the exposure information. Switching the req_type allows us to keep that code
         # in one place
         $req_type = "bydiff";
-        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose);
         return $results;
     }
@@ -120 +123 @@
         # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or 
         # the image from the diffRun
-        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose);
         if (!$results) {
             return undef;
         }
-        if ($img_type eq "diff") {
+        if ($stage eq "diff") {
             # lookup_diff has done all of the work
             return $results;
@@ -130 +133 @@
 
         my $image = $results->[0];
-        if ($img_type eq "raw") {
+        if ($stage eq "raw") {
             $req_type = "byid";
             $id = $image->{exp_id};
             return undef if !$id;
             # fall through and lookup byid
-        } elsif ($img_type eq "chip") {
+        } elsif ($stage eq "chip") {
             $req_type = "byid";
             $id = $image->{chip_id};
             return undef if !$id;
             # fall through and lookup byid
-        } elsif ($img_type eq "warp") {
+        } elsif ($stage eq "warp") {
             $req_type = "byid";
             $id = $image->{warp_id};
@@ -146 +149 @@
             return undef if !$id;
             # fall through and lookup by warp_id
-        } elsif ($img_type eq "stack") {
+        } elsif ($stage eq "stack") {
             $req_type = "byid";
             $id = $image->{stack_id};
@@ -153 +156 @@
         } else {
             # This is checked this elsewhere?
-            print STDERR "Error: $img_type is an unknown image type\n";
+            print STDERR "Error: $stage is an unknown image type\n";
             return undef;
         }
     } elsif ($req_type eq "byskycell") {
-        if (($img_type eq "raw") or ($img_type eq "chip")) {
+        if (($stage eq "raw") or ($stage eq "chip")) {
             print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n";
             return undef;
@@ -167 +170 @@
     }
 
-    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component,
-        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
+        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+        undef, undef, undef, $verbose);
+
+    return $results;
+}
+
+sub locate_images_for_row {
+    my $ipprc    = shift; 
+    my $imagedb  = shift;
+    my $camera   = shift;
+    my $row      = shift;
+    my $verbose  = shift;
+
+    my $req_type  = $row->{REQ_TYPE};
+
+    # we die in response to bad data in request files
+    # The caller is responsible for updating the database
+    # pstampparse.pl error checks now so this shouldn't happen
+    &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) 
+        if ($req_type ne "byid") and
+           ($req_type ne "byexp") and 
+           ($req_type ne "bycoord") and 
+           ($req_type ne "bydiff") and 
+           ($req_type ne "byskycell");
+
+    my $stage     = $row->{IMG_TYPE};
+    my $id        = $row->{ID};
+    my $component = $row->{COMPONENT};
+    my $tess_id   = $row->{TESS_ID};
+
+    my $filter    = $row->{REQFILT};
+    my $mjd_min   = $row->{MJD_MIN};
+    my $mjd_max   = $row->{MJD_MAX};
+    my $data_group = $row->{DATA_GROUP};
+    if (isnull($data_group) and !$default_data_groups) {
+        $default_data_groups = load_data_groups($verbose);
+    }
+
+    my $rownum     = $row->{ROWNUM};
+    my $job_type   = $row->{JOB_TYPE};
+    my $option_mask= $row->{OPTION_MASK};
+
+    my $dateobs_begin;
+    my $dateobs_end;
+    if (!iszero($mjd_min)) {
+        $dateobs_begin = mjd_to_dateobs($mjd_min);
+    }
+    if (!iszero($mjd_max)) {
+        $dateobs_end = mjd_to_dateobs($mjd_max);
+    }
+    if (isnull($tess_id)) {
+        $tess_id = undef;
+    }
+    if (isnull($data_group)) {
+        $data_group = undef;
+    }
+    if (isnull($filter)) {
+        $filter = undef;
+    }
+
+    if ($req_type eq "bycoord") {
+        my $x = $row->{CENTER_X};
+        my $y = $row->{CENTER_Y};
+        my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, 0, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+        return $results;
+    }
+
+    # for compatability with the lower level functions that haven't been converted to take a single row
+    my $rowList = [$row];
+
+    if (($req_type eq "byid") and ($stage eq "diff")) {
+        # lookups of all of the information for diff images requires a two level lookup to
+        # get the exposure information. Switching the req_type allows us to keep that code
+        # in one place
+        $req_type = "bydiff";
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose);
+        return $results;
+    }
+
+    my ($release_name, $survey) = get_release_info($row);
+
+    if ($req_type eq "bydiff") {
+        # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or 
+        # the image from the diffRun
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose);
+        if (!$results) {
+            return undef;
+        }
+        if ($stage eq "diff") {
+            # lookup_diff has done all of the work
+            return $results;
+        } 
+
+        my $image = $results->[0];
+        if ($stage eq "raw") {
+            $req_type = "byid";
+            $id = $image->{exp_id};
+            return undef if !$id;
+            # fall through and lookup byid
+        } elsif ($stage eq "chip") {
+            $req_type = "byid";
+            $id = $image->{chip_id};
+            return undef if !$id;
+            # fall through and lookup byid
+        } elsif ($stage eq "warp") {
+            $req_type = "byid";
+            $id = $image->{warp_id};
+            $component = $image->{skycell_id};
+            return undef if !$id;
+            # fall through and lookup by warp_id
+        } elsif ($stage eq "stack") {
+            $req_type = "byid";
+            $id = $image->{stack_id};
+            return undef if !$id;
+            # fall though and lookup by stack_id
+        } else {
+            # This is checked this elsewhere?
+            print STDERR "Error: $stage is an unknown image type\n";
+            return undef;
+        }
+    } elsif ($req_type eq "byskycell") {
+        if (($stage eq "raw") or ($stage eq "chip")) {
+            print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n";
+            return undef;
+        }
+        if (!$tess_id or !$component) {
+            print STDERR "component and tess_id are required for REQ_TYPE byskycell\n";
+            return undef;
+        }
+    }
+
+    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component,
+        0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
+        undef, $release_name, $survey,
+        $verbose);
 
     return $results;
@@ -193 +330 @@
     my $mjd_min = shift;
     my $mjd_max = shift;
+    my $selectedAstrom = shift;
+    my $release_name = shift;
+    my $survey = shift;
     my $verbose  = shift;
 
@@ -217 +357 @@
     my $magic_arg = $need_magic ? " -destreaked" : "";
 
+    my $row = $rowList->[0];
+
     # all rows have the same center type
-    my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+    my $skycenter = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+
+    my $use_releasetool = 0;
+    my $release_args;
+    if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) {
+        unless (isnull($release_name) and isnull($survey)) {
+            $use_releasetool = 1;
+            $release_args = " -release_name $release_name" if $release_name;
+            $release_args .= " -surveyName $survey" if $survey;
+        }
+    }
 
     my $component_args;
@@ -249 +401 @@
         # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then
         # choose the chips containing the center pixel by calling selectComponets() below
-        if ($component or $use_imfile_id or !$skycenter) {
-            $command = "$chiptool -processedimfile -pstamp_order -dbname $imagedb";
-            if ($component and $component ne 'all') {
-                $class_id = $component;
-                $component_args = " -class_id $class_id";
-            }
-        } else {
-            $command = "$chiptool -listrun -pstamp_order -dbname $imagedb";
+        if ($use_releasetool) {
+            $command = "$releasetool -dbname $imagedb -priority_order -listrelexp $release_args";
             $choose_components = 1;
-        }
-        $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        } else {
+            $command = "$chiptool -pstamp_order -dbname $imagedb";
+            if ($component or $use_imfile_id or !$skycenter) {
+                $command .= " -processedimfile";
+                if ($component and $component ne 'all') {
+                    $class_id = $component;
+                    $component_args = " -class_id $class_id";
+                }
+            } else {
+                $command .= " -listrun";
+                $choose_components = 1;
+            }
+            $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        }
         # XXX: perhaps we should stop resolving images to this level here and do it at job run time.
         # With the mdc file it has all of the information that it needs.
@@ -273 +431 @@
         $want_astrom  = 1;
         $set_class_id = 1;
-    } elsif ($stage eq "warp") {
-        if ($component or $use_imfile_id or !$skycenter) {
-            $command = "$warptool -warped -pstamp_order -dbname $imagedb";
-            if ($component and $component ne 'all') {
-                $skycell_id = $component;
-                $component_args = " -skycell_id $skycell_id";
-            }
-        } else {
-            $command = "$warptool -listrun -pstamp_order -dbname $imagedb";
+    } elsif ($stage eq 'warp') {
+        if ($use_releasetool) {
+            $command = "$releasetool -dbname $imagedb -priority_order -listrelexp $release_args";
             $choose_components = 1;
-        }
-        $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
+        } else {
+            $command = "$warptool -pstamp_order -dbname $imagedb";
+            if ($component or $use_imfile_id or !$skycenter) {
+                $command .= " -warped";
+                if ($component and $component ne 'all') {
+                    $skycell_id = $component;
+                    $component_args = " -skycell_id $skycell_id";
+                }
+            } else {
+                $command = " -listrun";
+                $choose_components = 1;
+            }
+            $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
+        }
         $image_name   = "PSWARP.OUTPUT";
         $mask_name    = "PSWARP.OUTPUT.MASK";
@@ -292 +456 @@
         $base_name    = "path_base"; # name of the field for the warptool output
     } elsif ($stage eq "diff") {
+        # XXX: is this path ever used for diff images anymore?
         if ($component or $use_imfile_id or !$skycenter) {
             $command = "$difftool -diffskyfile -pstamp_order -dbname $imagedb";
@@ -312 +477 @@
     } elsif ($stage eq "stack") {
         $skycell_id = $component;
-        $command = "$stacktool -sumskyfile -dbname $imagedb";
-        # only consider stacks in full state.
-        $command .= " -state full";
-        $id_opt = "-stack_id";
+        if ($use_releasetool) {
+            $command = "$releasetool -listrelstack -dbname $imagedb $release_args";
+            $base_name = 'stack_path_base';
+        } else {
+            $command = "$stacktool -sumskyfile -dbname $imagedb";
+            # only consider stacks in full state.
+            $command .= " -state full";
+            $id_opt = "-stack_id";
+            $base_name = 'path_base';
+        }
         $component_args = " -skycell_id $skycell_id" if $skycell_id and $skycell_id ne  'all';
 
@@ -329 +500 @@
         # this is wrong but gets the right answer. Need to figure out how to find the
         # rule properly
-        #$cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
         $cmf_name     = "PSWARP.OUTPUT.SOURCES";
+        # $cmf_name    = "PSPHOT.OUTPUT";    # this puts .fpa. in the name
         $psf_name    = "PPSTACK.TARGET.PSF";
-        $base_name   = "path_base";
     } else {
-        die "Unknown IMG_TYPe supplied: $stage";
+        die "Unknown IMG_TYPE supplied: $stage";
     }
 
@@ -377 +547 @@
     if (!isnull($data_group)) {
         $command .= " -data_group $data_group";
-    } elsif ($req_type eq 'byskycell') {
+    } elsif (!$use_releasetool and $req_type eq 'byskycell') {
+        # XXX: Why am I using default data_groups only for byskycell requests?
         $command .= $default_data_groups;
     }
@@ -392 +563 @@
         $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose);
     }
+
     if ($choose_components) {
         # the list of "images" is actually a list of "Runs"
         # match the coords in the rows to the components in the runs
         # returns an actual list of images
-        $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+        if ($skycenter) {
+            $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+        } else {
+            $images = selectComponentsByName($ipprc, $imagedb, $req_type, $stage, $component, $rowList, $images, $verbose);
+        }
     } else {
-        # put index for each of the rows into all of the the images
+        # put index for each of the rows into all of the the selected images
         setRowRefs($rowList, $images);
     }
@@ -462 +638 @@
             # dquery wants these set
             $out->{magicked} = 0;
-            $out->{data_state} = $out->{state};
+            if ($use_releasetool) {
+                if ($image->{skycal_path_base}) {
+                    $out->{astrom} = $image->{skycal_path_base} . ".cmf";
+                }
+                $out->{data_state} = $out->{state} = $out->{stack_state};
+            } else {
+                $out->{data_state} = $out->{state};
+            }
         }
         $out->{stage_id} = $stage_id;
@@ -470 +653 @@
         my $mask_base;
         if ($want_astrom) {
-            if (! defined $out->{astrom}) {
+            if ($selectedAstrom) {
+                $out->{astrom} = $selectedAstrom;
+                $out->{cam_path_base} = basename($selectedAstrom);
+            } elsif (! defined $out->{astrom}) {
                 if (! find_astrometry($ipprc, $imagedb, $out, $verbose)) {
                     print STDERR "failed to find astrometry for $stage $stage_id\n";
@@ -505 +691 @@
     my $byid     = shift;
     my $option_mask  = shift;
-    my $img_type = shift;
+    my $stage = shift;
     my $verbose = shift;
 
@@ -573 +759 @@
             # we have a stack - stack diff (well it might be stack - warp....)
             # but at any rate we only handle image type diff and stack
-            if (($img_type ne "diff") and ($img_type ne "stack")) {
-                print STDERR "lookup_diff: cannot lookup IMG_TYPE $img_type bydiff from a stack stack diff run\n";
+            if (($stage ne "diff") and ($stage ne "stack")) {
+                print STDERR "lookup_diff: cannot lookup IMG_TYPE $stage bydiff from a stack stack diff run\n";
                 setErrorCodes($rowList, $PSTAMP_INVALID_REQUEST);
                 # all images will be the same so we can stop
@@ -625 +811 @@
         }
 
-        if ($img_type eq "diff") {
+        if ($stage eq "diff") {
             my @imageList = ($image);
 
@@ -656 +842 @@
 sub lookup_bycoord {
     my $ipprc      = shift;
-    my $rowList    = shift;
+    my $row        = shift;
     my $imagedb    = shift;
-    my $img_type   = shift;
+    my $stage      = shift;
     my $tess_id    = shift;
     my $component  = shift;
@@ -671 +857 @@
     my $mjd_min = shift;
     my $mjd_max = shift;
+    my $astrom = shift;
     my $verbose    = shift;
 
+    my ($release_name, $survey) = get_release_info($row);
+
+    my $rowList = [$row];
+
     my $results = ();
-    if (($img_type eq "raw") or ($img_type eq "chip")) {
-
-        my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $img_type,
-            $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose);
+    if (($stage eq "raw") or ($stage eq "chip")) {
+
+        my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $stage,
+            $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose, $release_name, $survey, $row->{FWHM_MIN}, $row->{FWHM_MAX});
 
         if (!$chips or scalar @$chips == 0) {
@@ -684 +875 @@
             foreach my $chip (@$chips) {
                 next if $component and ($chip->{component} ne $component);
-                my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $chip->{id},
+                my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id},
                     $tess_id, $chip->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
+                    $chip->{astrom}, undef, undef,
+                    $verbose);
 
                 next if !$these_results;
@@ -695 +888 @@
     } else {
         # this should have been checked elsewhere
-        die "unexpected image type $img_type" if ($img_type ne "warp") 
-                                              and ($img_type ne "stack") and ($img_type ne "diff");
+        die "unexpected image type $stage" if ($stage ne "warp") 
+                                              and ($stage ne "stack") and ($stage ne "diff");
 
         my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
@@ -704 +897 @@
         } else {
             foreach my $skycell (@$skycells) {
-                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef,
+                my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
                     $skycell->{tess_id}, $skycell->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
+                    undef, $release_name, $survey,
+                    $verbose);
 
                 next if !$these_results;
@@ -722 +917 @@
 
 sub lookup_by_cam_id_and_coords {
-    my $ipprc      = shift;
-    my $imagedb   = shift;
-    my $img_type   = shift;
-    my $ra         = shift;
-    my $dec        = shift;
-    my $need_magic = shift;
+    my $ipprc       = shift;
+    my $imagedb     = shift;
+    my $stage       = shift;
+    my $ra          = shift;
+    my $dec         = shift;
+    my $need_magic  = shift;
     my $dateobs_begin  = shift;
     my $dateobs_end = shift;
@@ -733 +928 @@
     my $data_group = shift;
     my $verbose    = shift;
-
-    my $camruns;
-
-    {
-        my $search_radius = 1.6; # XXX: this should be camera specific
-
+    my $release_name = shift;
+    my $survey      = shift;
+    my $fwhm_min    = shift;
+    my $fwhm_max    = shift;
+
+    my $search_radius = 1.6; # XXX: this should be camera specific
+
+    my $command;
+    my $using_camtool = 0;
+    if ($release_name or $survey) {
+        $command = "$releasetool -dbname $imagedb -listrelexp";
+        if (!isnull($survey)) {
+            $command .= " -survey_name $survey";
+        }
+        if (!isnull($release_name)) {
+            $command .= " -release_name $release_name";
+        } else {
+            $command .= " -priority_order";
+        }
+        if ($fwhm_min != 0) {
+            $command .= " -fwhm_min $fwhm_min";
+        }
+        if ($fwhm_max != 0) {
+            $command .= " -fwhm_max $fwhm_max";
+        }
+    } else {
+        $using_camtool = 1;
         my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
-           $command .= " -ra $ra -decl $dec -radius $search_radius";
-           $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
-           $command .= " -dateobs_end   $dateobs_end"   if $dateobs_end  ;
-           $command .= " -filter $filter" if $filter;
            # NOTE: we are applying the data_group to the camera run.
            # If we're looking for chip stage images there is no guarentee that 
@@ -755 +967 @@
            }
            $command .= " -destreaked" if $need_magic;
-
-        # run the tool and parse the output
-        $camruns = runToolAndParse($command, $verbose);
-    }
-    if (!$camruns) {
+    }
+    $command .= " -ra $ra -decl $dec -radius $search_radius";
+    $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
+    $command .= " -dateobs_end   $dateobs_end"   if $dateobs_end  ;
+    $command .= " -filter $filter" if $filter;
+
+    # run the tool and parse the output
+    my $results = runToolAndParse($command, $verbose);
+    if (!$results) {
         return undef;
     }
     my $components;
     my $last_exp_id = 0;
-    foreach my $camRun (@$camruns) {
-        my $cam_id = $camRun->{cam_id};
-        my $exp_id = $camRun->{exp_id};
-
+    foreach my $result (@$results) {
+        my $cam_id = $result->{cam_id};
+        my $exp_id = $result->{exp_id};
+
+        # go on if we already have a result for this exposure
         next if $exp_id eq $last_exp_id;
 
-        next if $camRun->{quality};
-        next if $camRun->{fault};
-
-        updateCamRunCache($camRun);
+        next if $result->{quality};
+
+        if ($using_camtool) {
+            next if $result->{fault};
+        } else {
+            $result->{path_base} = $result->{cam_path_base};
+        }
+        updateCamRunCache($result);
 
         $last_exp_id = $exp_id;
 
         # XXX Use file rule
-        my $astrom = $camRun->{path_base} . ".smf";
+        my $astrom = $result->{path_base} . ".smf";
         my $astrom_resolved = $ipprc->file_resolve($astrom);
         next if !$astrom_resolved;
@@ -832 +1053 @@
         }
 
-        print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n";
+        print "cam_id $cam_id exp_id $result->{exp_id} chip_id $result->{chip_id} $class_id\n";
 
         # build the hash to return
@@ -840 +1061 @@
         # problem
         my $comp = {
-            exp_id    => $camRun->{exp_id},
-            exp_name  => $camRun->{exp_name},
-            chip_id   => $camRun->{chip_id},
-            cam_id    => $camRun->{cam_id},
+            exp_id    => $result->{exp_id},
+            exp_name  => $result->{exp_name},
+            chip_id   => $result->{chip_id},
+            cam_id    => $result->{cam_id},
             class_id  => $class_id,
-            component => $class_id
+            component => $class_id,
+            astrom    => $astrom
         };
-        if ($img_type eq "chip") {
-            $comp->{id} = $camRun->{chip_id};
-            $comp->{state} = $camRun->{chip_state};
-            $comp->{magicked} = $camRun->{chip_magicked};
-        } else {
-            $comp->{id} = $camRun->{exp_id};
+        if ($stage eq "chip") {
+            $comp->{id} = $result->{chip_id};
+            $comp->{state} = $result->{chip_state};
+            $comp->{magicked} = 0;
+        } else {
+            $comp->{id} = $result->{exp_id};
             $comp->{state} = 'full';
-            $comp->{magicked} = $camRun->{raw_magicked};
+            $comp->{magicked} = $0;
         }
         push @$components, $comp;
@@ -1281 +1503 @@
 }
 
+sub selectComponentsByName {
+    my $ipprc = shift;
+    my $imagedb = shift;
+    my $req_type = shift;
+    my $stage  = shift;
+    my $component  = shift;
+    my $rowList = shift;
+    my $runList = shift;
+    my $verbose = shift;
+    my $results = [];
+
+    $component = "" if $component and lc($component) eq 'all';
+
+    # all runs match all rows so they can share a common row_index
+    # XXX: I'm not sure we ever actually get here with more than one
+    # row
+    my $numRows = scalar @$rowList;
+    my $row_index = [];
+    for (my $i=0; $i < $numRows; $i++) {
+        push @$row_index, $i;
+    }
+    
+    if (($req_type eq "byid") or ($req_type eq "byexp")) {
+        my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", "");
+        foreach my $run (@$runList) {
+            # now find the images for this run
+            #foreach my $c (keys %components) {
+            my $command;
+            my $stage_id;
+            if ($stage eq 'raw') {
+                $stage_id = $run->{exp_id};
+                $command = "$regtool -processedimfile -exp_id $stage_id";
+                $command .= " -class_id $component" if $component;
+            } elsif ($stage eq 'chip') {
+                $stage_id = $run->{chip_id};
+                $command = "$chiptool -processedimfile -chip_id $stage_id";
+                $command .= " -class_id $component" if $component;
+            } elsif ($stage eq 'warp') {
+                $stage_id = $run->{warp_id};
+                $command = "$warptool -warped -warp_id $stage_id";
+                $command .= " -skycell_id $component" if $component;
+            }
+            $command .= " -dbname $imagedb";
+            my $images = runToolAndParse($command, $verbose);
+            if (!defined $images) {
+                print "No  $component found for $stage ${stage}_id $stage_id\n";
+                next;
+            }
+            foreach my $image (@$images) {
+                $image->{row_index} = $row_index;
+                if (($stage eq "raw") or ($stage eq 'chip')) {
+                    $image->{cam_id} = $run->{cam_id};
+                    $image->{cam_path_base} = $run->{cam_path_base};
+                }
+                push @$results, $image;
+            }
+        }
+    } else {
+        &my_die ("selectComponentsByName not supported for REQ_TYPE: $req_type\n", $PS_EXIT_PROG_ERROR);
+    }
+    return $results;
+}
+
 sub filterRuns {
     my $stage      = shift;
@@ -1407 +1692 @@
     $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" 
                                                                 and $missing_tools = 1);
+    $releasetool = can_run('releasetool') or (warn "Can't find releasetool" 
+                                                                and $missing_tools = 1);
     $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
     $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
@@ -1462 +1749 @@
 }
 
+sub get_release_info {
+    my $row = shift;
+
+    my $survey;
+    my $release_name = $row->{IPP_RELEASE};
+    if (isnull($release_name)) {
+        $release_name = "";
+        # no release check for survey
+        $survey = $row->{SURVEY_NAME};
+    } else {
+        $survey = "";
+    }
+
+    return ($release_name, $survey);
+}
+
 sub my_die
 {

branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -52 +52 @@
                     $PSTAMP_NO_OVERLAP
                     $PSTAMP_NOT_AUTHORIZED
+                    $PSTAMP_UNKNOWN_PROJECT
                     );
 our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
@@ -101 +102 @@
 our $PSTAMP_NO_OVERLAP       = 28;
 our $PSTAMP_NOT_AUTHORIZED   = 29;
+our $PSTAMP_UNKNOWN_PROJECT  = 22;
 
 
@@ -167 +169 @@
     my $fields_output;
     {
-        my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USERNAME EMAIL";
+        my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
@@ -187 +189 @@
     if ($extver > 1) {
         $header{ACTION} = $action;
-        $header{USER} = $username;
         $header{EMAIL} = $email;
     } else {
         $header{ACTION} = $action = "PROCESS";
-        $header{USERNAME} = 'null';
         $header{EMAIL} = 'null';
     }
 
-    if ($action eq "LIST") {
-        return (\%header, undef);
-    } elsif ($action ne "PROCESS") {
-        die "unexpected request ACTION found: $action in $request_file_name";
+    if ($action ne "PROCESS" and $action ne 'PREVIEW') {
+        die "\nunexpected request ACTION found: $action in $request_file_name";
     }
 

branches/eam_branches/ipp-20130307/ippMonitor/raw/czartool_getplot.php

@@ -38 +38 @@
 else if ($type=="hp_s")
 $filePath = "$path/heathers_plots/skycal.png";
-
+else if ($type=="hp_cm")
+$filePath = "$path/heathers_plots/cam_merged.png";
+else if ($type=="hp_sm")
+$filePath = "$path/heathers_plots/skycal_merged.png";
 
 $file = fopen ($filePath, "r");

branches/eam_branches/ipp-20130307/ippMonitor/raw/heathers_plot.php

@@ -10 +10 @@
 menu('ipp.czar.dat', 'useful tools for the czar', 'ipp.css', $ID['link'], $ID['proj']);
 
-echo "These images are generated every day at 7:00 HST (17:00 GMT)\n";
+echo "These images are generated every day at 7:00 HST (17:00 GMT) on ipp@ippc11\n";
 // document body
 echo "<table>\n";
@@ -23 +23 @@
 echo "</tr>\n";
 
+echo "<tr>\n";
+echo "<td>Camera Merged</td>\n";
+echo "<td>SkyCal Merged</td>\n";
+echo "</tr>\n";
+
+echo "<tr>\n";
+echo "<td><img src=\"czartool_getplot.php?type=hp_cm\"/></td>\n";
+echo "<td><img src=\"czartool_getplot.php?type=hp_sm\"/></td>\n";
+echo "</tr>\n";
+
 echo "</table>\n";
 

branches/eam_branches/ipp-20130307/ippTasks/release.pro

@@ -129 +129 @@
     host anyhost
 
-    $run = relgroup_exp_list.pl --group_id $GROUP_ID --group_type $GROUP_TYPE --release_name $RELEASE_NAME
+    $run = relgroup_exp_list.pl --group_id $GROUP_ID --group_type $GROUP_TYPE --release_name $RELEASE_NAME --rebuild
     if ("$GROUP_TYPE" == "lap") 
         $run = $run --lap_id $LAP_ID

branches/eam_branches/ipp-20130307/ippTasks/survey.pro

@@ -19 +19 @@
  book create SURVEY_SKYCAL
  book create SURVEY_LAPGROUP
+ book create SURVEY_RELEXP
  $haveSurveyBooks = TRUE
 end
@@ -35 +36 @@
 $SURVEY_SKYCAL_DB = 0
 $SURVEY_LAPGROUP_DB = 0
+$SURVEY_RELEXP_DB = 0
 $SURVEY_STATICSKYSINGLE_DB = 0
 $SURVEY_EXEC = 120
@@ -83 +85 @@
     active true
   end
+  task survey.relexp
+    active true
+  end
 end
 
@@ -126 +131 @@
   end
   task survey.lapgroup
+    active false
+  end
+  task survey.relexp
     active false
   end
@@ -533 +541 @@
   end
   book listbook SURVEY_LAPGROUP
+end
+
+macro survey.add.relexp
+  if ($0 != 3)
+    echo "USAGE: survey.add.relexp (label) (releasename)"
+    break
+  end
+  book newpage SURVEY_RELEXP $1
+  book setword SURVEY_RELEXP $1 LABEL $1
+  book setword SURVEY_RELEXP $1 RELEASE_NAME $2
+  book setword SURVEY_RELEXP $1 STATE PENDING
+end
+
+macro survey.del.relexp
+  if ($0 != 2)
+    echo "USAGE: survey.del.relexp (label)"
+    break
+  end
+  book delpage SURVEY_RELEXP $1
+end
+
+macro survey.show.relexp
+  if ($0 != 1)
+    echo "USAGE: survey.show.relexp"
+    break
+  end
+  book listbook SURVEY_RELEXP
 end
 
@@ -1626 +1661 @@
 
       # Select different database
-      $SURVEY_LAPGROUP ++
-      if ($SURVEY_LAPGROUP >= $DB:n) set SURVEY_LAPGROUP = 0
+      $SURVEY_LAPGROUP_DB ++
+      if ($SURVEY_LAPGROUP_DB >= $DB:n) set SURVEY_LAPGROUP_DB = 0
     end
 
@@ -1633 +1668 @@
     book getword SURVEY_LAPGROUP $label SEQ_ID -var SEQ_ID
 
-    # For now the list of filters
+    # For now the list of filters is hardcoded here
     $run = laptool -definegroup -seq_id $SEQ_ID -set_label $label -filter g.00000 -filter r.00000 -filter i.00000 -filter z.00000 -filter y.00000
 
@@ -1666 +1701 @@
   end
 end
+
+task survey.relexp
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.relexp.log
+  stderr $LOGDIR/survey.relexp.log
+
+  task.exec
+    book npages SURVEY_RELEXP -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_RELEXP 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_RELEXP $i -var label
+        book setword SURVEY_RELEXP $label STATE NEW
+      end
+      book getpage SURVEY_RELEXP 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_RELEXP_DB ++
+      if ($SURVEY_RELEXP_DB >= $DB:n) set SURVEY_RELEXP_DB = 0
+    end
+
+    book setword SURVEY_RELEXP $label STATE DONE
+    book getword SURVEY_RELEXP $label RELEASE_NAME -var RELEASE_NAME
+
+    $run = releasetool -definerelexp -label $label -release_name $RELEASE_NAME -set_state processed
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_RELEXP_DB
+      option $DB:$SURVEY_RELEXP_DB
+    end
+    
+    # echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

branches/eam_branches/ipp-20130307/ippconfig/dvo.photcodes

@@ -165 +165 @@
   3004  SYNTH.z              ref   0.000  0.000 0.000     -     - 0.0000     0     4     nan 0.000 2.000  0.600   0x0000 0x0000  0x0000 0x0000
   3005  SYNTH.y              ref   0.000  0.000 0.000     -     - 0.0000     0     5     nan 0.000 2.000  0.600   0x0000 0x0000  0x0000 0x0000
+  3006  SYNTH.w              ref   0.000  0.000 0.000     -     - 0.0000     0     9     nan 0.000 2.000  0.600   0x0000 0x0000  0x0000 0x0000
 
   100   MEGACAM.u.00         dep  25.280 -0.350 0.000     -     - 0.0000     0     -   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000
@@ -836 +837 @@
   10475 GPC1.y.XY75          dep  23.320 -0.073 0.000     -     - 0.0000     0     5   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
   10476 GPC1.y.XY76          dep  23.320 -0.073 0.000     -     - 0.0000     0     5   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10501 GPC1.w.XY01          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10502 GPC1.w.XY02          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10503 GPC1.w.XY03          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10504 GPC1.w.XY04          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10505 GPC1.w.XY05          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10506 GPC1.w.XY06          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10510 GPC1.w.XY10          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10511 GPC1.w.XY11          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10512 GPC1.w.XY12          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10513 GPC1.w.XY13          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10514 GPC1.w.XY14          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10515 GPC1.w.XY15          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10516 GPC1.w.XY16          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10517 GPC1.w.XY17          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10520 GPC1.w.XY20          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10521 GPC1.w.XY21          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10522 GPC1.w.XY22          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10523 GPC1.w.XY23          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10524 GPC1.w.XY24          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10525 GPC1.w.XY25          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10526 GPC1.w.XY26          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10527 GPC1.w.XY27          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10530 GPC1.w.XY30          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10531 GPC1.w.XY31          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10532 GPC1.w.XY32          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10533 GPC1.w.XY33          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10534 GPC1.w.XY34          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10535 GPC1.w.XY35          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10536 GPC1.w.XY36          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10537 GPC1.w.XY37          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10540 GPC1.w.XY40          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10541 GPC1.w.XY41          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10542 GPC1.w.XY42          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10543 GPC1.w.XY43          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10544 GPC1.w.XY44          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10545 GPC1.w.XY45          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10546 GPC1.w.XY46          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10547 GPC1.w.XY47          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10550 GPC1.w.XY50          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10551 GPC1.w.XY51          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10552 GPC1.w.XY52          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10553 GPC1.w.XY53          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10554 GPC1.w.XY54          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10555 GPC1.w.XY55          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10556 GPC1.w.XY56          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10557 GPC1.w.XY57          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10560 GPC1.w.XY60          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10561 GPC1.w.XY61          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10562 GPC1.w.XY62          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10563 GPC1.w.XY63          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10564 GPC1.w.XY64          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10565 GPC1.w.XY65          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10566 GPC1.w.XY66          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10567 GPC1.w.XY67          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10571 GPC1.w.XY71          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10572 GPC1.w.XY72          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10573 GPC1.w.XY73          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10574 GPC1.w.XY74          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10575 GPC1.w.XY75          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
-  10576 GPC1.w.XY76          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10501 GPC1.w.XY01          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10502 GPC1.w.XY02          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10503 GPC1.w.XY03          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10504 GPC1.w.XY04          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10505 GPC1.w.XY05          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10506 GPC1.w.XY06          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10510 GPC1.w.XY10          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10511 GPC1.w.XY11          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10512 GPC1.w.XY12          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10513 GPC1.w.XY13          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10514 GPC1.w.XY14          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10515 GPC1.w.XY15          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10516 GPC1.w.XY16          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10517 GPC1.w.XY17          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10520 GPC1.w.XY20          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10521 GPC1.w.XY21          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10522 GPC1.w.XY22          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10523 GPC1.w.XY23          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10524 GPC1.w.XY24          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10525 GPC1.w.XY25          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10526 GPC1.w.XY26          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10527 GPC1.w.XY27          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10530 GPC1.w.XY30          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10531 GPC1.w.XY31          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10532 GPC1.w.XY32          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10533 GPC1.w.XY33          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10534 GPC1.w.XY34          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10535 GPC1.w.XY35          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10536 GPC1.w.XY36          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10537 GPC1.w.XY37          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10540 GPC1.w.XY40          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10541 GPC1.w.XY41          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10542 GPC1.w.XY42          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10543 GPC1.w.XY43          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10544 GPC1.w.XY44          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10545 GPC1.w.XY45          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10546 GPC1.w.XY46          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10547 GPC1.w.XY47          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10550 GPC1.w.XY50          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10551 GPC1.w.XY51          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10552 GPC1.w.XY52          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10553 GPC1.w.XY53          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10554 GPC1.w.XY54          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10555 GPC1.w.XY55          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10556 GPC1.w.XY56          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10557 GPC1.w.XY57          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10560 GPC1.w.XY60          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10561 GPC1.w.XY61          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10562 GPC1.w.XY62          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10563 GPC1.w.XY63          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10564 GPC1.w.XY64          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10565 GPC1.w.XY65          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10566 GPC1.w.XY66          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10567 GPC1.w.XY67          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10571 GPC1.w.XY71          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10572 GPC1.w.XY72          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10573 GPC1.w.XY73          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10574 GPC1.w.XY74          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10575 GPC1.w.XY75          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10576 GPC1.w.XY76          dep  26.060  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
   11000 GPC1.g.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
   11100 GPC1.r.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000

branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppStack.config

@@ -1 +1 @@
 PSF.MODEL       STR     PS_MODEL_PS1_V1 # Model for PSF generation
+BACKGROUND.MODEL BOOL   F
 
 ZP.AIRMASS      METADATA                # Airmass terms by filter
@@ -24 +25 @@
     PSF.INPUT.MAX           F32   6.0   # never really want >6 pixels in refstack
     PSF.INPUT.CLIP.NSIGMA   F32   1.0   # sample typically on rising side of distribution (set even smaller?)
+    PSF.INPUT.THRESH        F32   0.0
+    PSF.INPUT.ASYMMETRY     F32   0.0
 END
 
@@ -30 +33 @@
     OUTPUT.LOGFLUX          BOOL  FALSE
     OUTPUT.REPLICATE        BOOL  TRUE
-    PSF.INPUT.MAX           F32   7.5
+    PSF.INPUT.MAX           F32   10.0
     PSF.INPUT.CLIP.NSIGMA   F32   1.5
+    PSF.INPUT.THRESH        F32   5.0
+    PSF.INPUT.ASYMMETRY     F32   0.2
 END
 
@@ -40 +45 @@
     OUTPUT.NOCOMP          BOOL  TRUE
     OUTPUT.LOGFLUX         BOOL  FALSE
-    PSF.INPUT.MAX          F32   7.5    # since not convolved limit, relax/tune?
+    PSF.INPUT.MAX          F32   10.0    # since not convolved limit, relax/tune?
     PSF.INPUT.CLIP.NSIGMA  F32   1.5
+    PSF.INPUT.THRESH        F32   5.0
+    PSF.INPUT.ASYMMETRY     F32   0.2
 END
 
@@ -52 +59 @@
     OUTPUT.NOCOMP           BOOL  FALSE
     OUTPUT.LOGFLUX          BOOL  TRUE
-    PSF.INPUT.MAX           F32   7.5   # input cut more restrictive now 12->7.5 pix, limit target PSF in convolved stacks 
+    PSF.INPUT.MAX           F32   10.0  # input cut more restrictive now 12->10 pix, limit target PSF in convolved stacks 
     PSF.INPUT.CLIP.NSIGMA   F32   1.5   # additional cut if majority of inputs much smaller than INPUT.MAX
+    PSF.INPUT.THRESH        F32   5.0
+    PSF.INPUT.ASYMMETRY     F32   0.2
 END
 

branches/eam_branches/ipp-20130307/ippconfig/isp/dvo.config

@@ -1 +1 @@
 
 # location of DVO database tables
-CATDIR                  /data/alala/eugene/isp/catdir
+#CATDIR                 /data/alala/eugene/isp/catdir
+MOSAICNAME              ISP-APOGEE
 
 # keywords used by DVO to interpret the headers

branches/eam_branches/ipp-20130307/ippconfig/isp/ppImage.config

@@ -623 +623 @@
   END   
 END
+
+PPIMAGE_BKGCONT    METADATA
+  OVERSCAN         BOOL    FALSE           # Overscan subtraction
+  NONLIN           BOOL    FALSE           # Non-linearity correction; not implemented
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    FALSE           # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    FALSE           # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  MASK.BUILD       BOOL    FALSE           # Build internal mask?
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BASE.FITS        BOOL    FALSE           # Save base image?
+  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image? 
+  BIN1.FITS        BOOL    FALSE           # Save 1st binned chip image?
+  BIN2.FITS        BOOL    FALSE           # Save 2nd binned chip image?
+  BIN1.JPEG        BOOL    FALSE           # Save 1st binned jpeg?
+  BIN2.JPEG        BOOL    FALSE           # Save 2nd binned jpeg?
+  BIN1.XBIN        S32     1               # Image is already binned
+  BIN1.YBIN        S32     1               # Image is already binned
+  BIN2.XBIN        S32     1               # Image is already binned
+  BIN2.YBIN        S32     1               # Image is already binned
+  MASK.SATURATED   BOOL    FALSE           # DO NOT Mask the saturated pixels
+  MASK.LOW         BOOL    FALSE           # DO NOT Mask the saturated pixels
+  BACKGROUND.CONTINUITY BOOL FALSE
+  PATTERN.CONTINUITY.WIDTH S32 2
+END
+

branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-mef.mdc

@@ -118 +118 @@
 PSWARP.VARIANCE         INPUT    @FILES        CHIP       VARIANCE
 PSWARP.MASK             INPUT    @FILES        CHIP       MASK
-PSWARP.SKYCELL          INPUT    @FILES        CHIP       IMAGE
-PSWARP.ASTROM           INPUT    @FILES        CHIP       CMF
-PSWARP.BKGMODEL           INPUT    @FILES        CHIP       IMAGE
+PSWARP.SKYCELL          INPUT    @FILES        CHIP       WCS
+PSWARP.ASTROM           INPUT    @FILES        CHIP       WCS
+PSWARP.BKGMODEL         INPUT    @FILES        CHIP       IMAGE
 
 ## files used by ppsub

branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-simple.mdc

@@ -88 +88 @@
 PSWARP.VARIANCE           INPUT    @FILES        FPA        VARIANCE
 PSWARP.MASK               INPUT    @FILES        FPA        MASK
-PSWARP.SKYCELL            INPUT    @FILES        FPA        IMAGE     
-PSWARP.ASTROM             INPUT    @FILES        FPA        CMF
+PSWARP.SKYCELL            INPUT    @FILES        FPA        WCS
+PSWARP.ASTROM             INPUT    @FILES        FPA        WCS
 PSWARP.BKGMODEL           INPUT    @FILES        CHIP       IMAGE
 

branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-split.mdc

@@ -106 +106 @@
 PSWARP.VARIANCE           INPUT    @FILES        CHIP       VARIANCE
 PSWARP.MASK               INPUT    @FILES        CHIP       MASK
-PSWARP.SKYCELL            INPUT    @FILES        FPA        IMAGE
-PSWARP.ASTROM             INPUT    @FILES        CHIP       CMF
+PSWARP.SKYCELL            INPUT    @FILES        CHIP       WCS
+PSWARP.ASTROM             INPUT    @FILES        CHIP       WCS
+#PSWARP.SKYCELL            INPUT    @FILES        CHIP       CMF
+#PSWARP.ASTROM             INPUT    @FILES        CHIP       CMF
 PSWARP.BKGMODEL           INPUT    @FILES        CHIP       IMAGE
                           

branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStack.config

@@ -92 +92 @@
 PSF.INPUT.MAX           F32  NAN
 PSF.INPUT.CLIP.NSIGMA   F32  NAN
+PSF.INPUT.THRESH        F32  NAN
+PSF.INPUT.ASYMMETRY     F32  NAN
 
 TEMP.IMAGE      STR     conv.im.fits    # Suffix for temporary convolved images

branches/eam_branches/ipp-20130307/ppImage/src/ppImageParseCamera.c

@@ -22 +22 @@
     ppImageOptions *options = ppImageOptionsParse(config);
 
-    // parse the header to extract out whether this OTA has video cells.
-    // CZW: 2012-05-01 I'm somewhat ashamed of what needs to be done to get to the header. There has to be a simpler way, right?
-/*     pmFPAview *view = pmFPAviewAlloc(0);// View for level of interest */
-/*     pmChip *chip;                       // Chip from FPA */
-/*     while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { */
-/*       if (!chip->process || !chip->file_exists) { */
-/*      continue; */
-/*       } */
-/* /\*       if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { *\/ */
-/* /\*  continue; *\/ */
-/* /\*       } *\/ */
-/*       psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */
-/*       if (psMetadataLookupStr(NULL,chip->hdu->header,"CELLMODE")) { */
-/*      psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */
-/*      char *Vptr = strchr(psMetadataLookupStr(NULL,chip->hdu->header,"CELLMODE"),'V'); */
-/*      if (Vptr) { */
-/*        options->hasVideo = true; */
-/*        psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */
-/*      } */
-/*       } */
-/*     } */
-/*     pmFPAviewReset(view); */
-/*     psFree(view); */
-    
     // the following are defined from the argument list, if given,
     // otherwise they revert to the config information

branches/eam_branches/ipp-20130307/ppStack/src/ppStackMatch.c

@@ -8 +8 @@
 // #define TESTING                         // Enable debugging output
 // #define TESTING_REUSE_CONV           // Enable debugging for re-using convolved outputs from previous run
+
+// #define TESTING_CZW 
 
 #ifdef TESTING_REUSE_CONV
@@ -264 +266 @@
             // Add the background into the target image
             psImage *bgImage = stackBackgroundModel(readout, config); // Image of background
-            psBinaryOp(fake->image, fake->image, "+", bgImage);
+            psBinaryOp(fake->image, fake->image, "+", bgImage);
             psFree(bgImage);
 #endif
 
-#ifdef TESTING
+#ifdef TESTING_CZW
             {
                 pmHDU *hdu = pmHDUFromCell(readout->parent);
                 psString name = NULL;
-                psStringAppend(&name, "fake_%03d.fits", index);
+                psStringAppend(&name, "/local/tmp/fake_%03d.fits", index);
                 pmStackVisualPlotTestImage(fake->image, name);
                 psFits *fits = psFitsOpen(name, "w");
@@ -282 +284 @@
                 pmHDU *hdu = pmHDUFromCell(readout->parent);
                 psString name = NULL;
-                psStringAppend(&name, "real_%03d.fits", index);
+                psStringAppend(&name, "/local/tmp/real_%03d.fits", index);
                 pmStackVisualPlotTestImage(readout->image, name);
                 psFits *fits = psFitsOpen(name, "w");
                 psFree(name);
                 psFitsWriteImage(fits, hdu->header, readout->image, 0, NULL);
+                psFitsClose(fits);
+            }
+            {
+                pmHDU *hdu = pmHDUFromCell(readout->parent);
+                psString name = NULL;
+                psStringAppend(&name, "/local/tmp/real_var_%03d.fits", index);
+                pmStackVisualPlotTestImage(readout->variance, name);
+                psFits *fits = psFitsOpen(name, "w");
+                psFree(name);
+                psFitsWriteImage(fits, hdu->header, readout->variance, 0, NULL);
                 psFitsClose(fits);
             }
@@ -355 +367 @@
 
 
-#ifdef TESTING
+#ifdef TESTING_CZW
             {
                 pmHDU *hdu = pmHDUFromCell(readout->parent);
                 psString name = NULL;
-                psStringAppend(&name, "conv_%03d.fits", index);
+                psStringAppend(&name, "/local/tmp/conv_%03d.fits", index);
                 pmStackVisualPlotTestImage(conv->image, name);
                 psFits *fits = psFitsOpen(name, "w");
@@ -369 +381 @@
                 pmHDU *hdu = pmHDUFromCell(readout->parent);
                 psString name = NULL;
-                psStringAppend(&name, "diff_%03d.fits", index);
+                psStringAppend(&name, "/local/tmp/conv_var_%03d.fits", index);
+                pmStackVisualPlotTestImage(conv->variance, name);
+                psFits *fits = psFitsOpen(name, "w");
+                psFree(name);
+                psFitsWriteImage(fits, hdu->header, conv->variance, 0, NULL);
+                psFitsClose(fits);
+            }
+            {
+                pmHDU *hdu = pmHDUFromCell(readout->parent);
+                psString name = NULL;
+                psStringAppend(&name, "/local/tmp/diff_%03d.fits", index);
                 pmStackVisualPlotTestImage(fake->image, name);
                 psFits *fits = psFitsOpen(name, "w");
@@ -510 +532 @@
     }
 
-    if (!stackRenormaliseReadout(config, readout)) {
-        psFree(rng);
-        psFree(bg);
-        return false;
-    }
-
+#ifdef TESTING_CZW
+    // CZW begin check of weight before renormalize step
     // Measure the variance level for the weighting
     if (psMetadataLookupBool(NULL, recipe, "WEIGHTS")) {
@@ -529 +547 @@
         options->weightings->data.F32[index] = 1.0;
     }
-    psLogMsg("ppStack", PS_LOG_INFO, "Weighting for image %d is %f\n",
-             index, options->weightings->data.F32[index]);
+    psLogMsg("ppStack", PS_LOG_INFO, "Prenorm Weighting for image %d is %g covar %g median %g\n",
+             index, options->weightings->data.F32[index],psImageCovarianceFactor(readout->covariance),
+             psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN));
+    // CZW end
+#endif
+    if (!stackRenormaliseReadout(config, readout)) {
+        psFree(rng);
+        psFree(bg);
+        return false;
+    }
+    
+    // Measure the variance level for the weighting
+    if (psMetadataLookupBool(NULL, recipe, "WEIGHTS")) {
+        if (!psImageBackground(bg, NULL, readout->variance, readout->mask, maskVal | maskBad, rng)) {
+            psError(PPSTACK_ERR_DATA, false, "Can't measure mean variance for image.");
+            psFree(rng);
+            psFree(bg);
+            return false;
+        }
+        options->weightings->data.F32[index] = 1.0 / (psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN) *
+                                                      psImageCovarianceFactor(readout->covariance));
+    } else {
+        options->weightings->data.F32[index] = 1.0;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Weighting for image %d is %g covar %g median %g\n",
+             index, options->weightings->data.F32[index],psImageCovarianceFactor(readout->covariance),
+             psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN));
 
     psFree(rng);

branches/eam_branches/ipp-20130307/ppStack/src/ppStackPrepare.c

@@ -1 +1 @@
 # include "ppStack.h"
+
+
+// Option to use the old FWHM rejection code.
+# define NEW_REJECTION
+// #define OLD_REJECTION
+
 
 # define RE_PHOTOMETER 0
@@ -253 +259 @@
     float maxFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.MAX"); // max allowed input fwhm
     float clipFWHMnSig = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.CLIP.NSIGMA"); // sigma clipping of inputs
-
+    float threshFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.THRESH"); // FWHM size that we refuse to clip below
+    float asymmetryFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.ASYMMETRY"); // max bimodal asymmetry
     psString log = psStringCopy("Input seeing FWHMs:\n"); // Log message
     bool havePSFs = false;                                // Do we have any PSFs?
@@ -286 +293 @@
             options->inputSeeing->data.F32[i] = sumFWHM / (float)numFWHM;
         }
-
+#ifdef OLD_REJECTION
         // reject any input images which exceed the specified max FWHM
         if (isfinite(maxFWHM) && (options->inputSeeing->data.F32[i] > maxFWHM)) {
@@ -292 +299 @@
             psLogMsg("ppStack", PS_LOG_INFO, "PSF FWHM for image %d is too large (%f vs %f maxFWHM) --- rejected.", i, options->inputSeeing->data.F32[i], maxFWHM);
         }
-
+        // End old rejection fixed limit.
+#endif
         psStringAppend(&log, "Input %d: %f\n", i, options->inputSeeing->data.F32[i]);
     }
@@ -300 +308 @@
     psFree(log);
 
+#ifdef NEW_REJECTION
+    // Do GMM test with two modes to decide where to put the break.
+    double Punimodal;
+    int m = 2;
+    psVector *modes = psVectorAlloc(num,PS_TYPE_F32);
+    psVector *means = psVectorAlloc(m,PS_TYPE_F32);
+    psVector *S     = psVectorAlloc(m,PS_TYPE_F32);
+    psVector *pi    = psVectorAlloc(m,PS_TYPE_F32);
+    psImage *P      = psImageAlloc(m,num,PS_TYPE_F32);
+    float limit;
+    if (!psMM1DClass(options->inputSeeing,
+                     options->inputSeeing->n,
+                     modes,means,
+                     S,pi,P,
+                     2,
+                     &Punimodal)) {
+      // Handle error here
+      psFree(modes);
+      psFree(means);
+      psFree(S);
+      psFree(pi);
+      psFree(P);
+      return(false);
+    }
+    //    fprintf(stderr,"means: %g %g\n",means->data.F32[0],means->data.F32[1]);
+    //    fprintf(stderr,"sigma: %g %g \n",S->data.F32[0],S->data.F32[1]);
+
+    //    fprintf(stderr,"pi:    %g %g\n",pi->data.F32[0],pi->data.F32[1]);
+    
+    //    unwrittenGMMfunction(options->inputSeeing, options->inputMask, 0xff,
+    //           &Punimodal,
+    //           &m1,&s1,&pi1,
+    //           &m2,&s2,&pi2);
+
+    // Use Gaussian mixture model analysis of the FWHM distribution to decide an optional FWHM limit
+    if ((Punimodal > 0.5)|| // This distribution is best represented by a single mode
+        (num <= 4)) {       // Or we have a small number of inputs, making this statistic poor.
+      limit = maxFWHM;
+    }
+    else {                  // This is a bimodal distribution
+      if ((fabs(pi->data.F32[0] - pi->data.F32[1]) < asymmetryFWHM)||  // However, both modes are equally populated
+          (pi->data.F32[1] > pi->data.F32[0])) {                       // Or the larger FWHM mode is more populated
+        limit = means->data.F32[1] + clipFWHMnSig * S->data.F32[1];
+      }
+      else {                                   // The smaller FWHM mode is more populated
+        limit = means->data.F32[0] + clipFWHMnSig * S->data.F32[0];
+      }
+    }
+    if (limit > maxFWHM)    { limit = maxFWHM; }    // We should not be larger than our max
+    if (limit < threshFWHM) { limit = threshFWHM; } // Nor smaller than our min
+
+    for (int i = 0; i < num; i++) {
+      if (options->inputSeeing->data.F32[i] > limit) {
+        options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_PSF;
+        psLogMsg("ppStack", PS_LOG_INFO,
+                 "PSF FWHM for image %d is too large (%f vs %f fwhmlimit) --- rejected",
+                 i, options->inputSeeing->data.F32[i],limit);
+      }
+    }
+    psFree(means);
+    psFree(modes);
+    psFree(S);
+    psFree(pi);
+    psFree(P);
+    // End new rejection code
+#endif
+    
+#ifdef OLD_REJECTION
     // We should have the ability to filter the input list based on the seeing:
     // * reject above some max value and/or min value
@@ -323 +399 @@
         }
     }
-
+    // End old rejection sigma clip
+#endif
     // Generate target PSF
     if (options->convolve) {

branches/eam_branches/ipp-20130307/ppStack/src/ppStackSources.c

@@ -11 +11 @@
 #endif
 
-#ifdef TESTING
+// #define TESTING_CZW
+
+#ifdef TESTING_CZW
 // Dump matches to a file
 static void dumpMatches(const char *filename, // File to which to dump
@@ -28 +30 @@
         psVectorInit(magErr, NAN);
         for (int j = 0; j < match->num; j++) {
+          if (match->mask) {
             if (match->mask->data.PS_TYPE_VECTOR_MASK_DATA[j]) {
-                continue;
-            }
+              continue;
+            }
+          }
             int index = match->image->data.U32[j]; // Image index
             mag->data.F32[index] = match->mag->data.F32[j] - zp->data.F32[index];
@@ -39 +43 @@
         }
         for (int j = 0; j < num; j++) {
-            fprintf(outMatches, "%f (%f) ", mag->data.F32[j], magErr->data.F32[j]);
+          fprintf(outMatches, "%f %f %f %f %f %f\t",
+                  match->mag->data.F32[j],
+                  zp->data.F32[j],
+                  trans ? trans->data.F32[j] : NAN,
+                  magErr->data.F32[j],
+                  match->x->data.F32[j],
+                  match->y->data.F32[j]);
         }
         fprintf(outMatches, "\n");
@@ -271 +281 @@
         return false;
     }
-
-#ifdef TESTING
+    options->sources = psMemIncrRefCounter(matches);
+    psArray *matchedSources = psArrayAlloc(0);
+
+#ifdef TESTING_CZW
     dumpMatches("source_match.dat", num, matches, zp, NULL);
 #endif
 
     if (options->matchZPs) {
-        psVector *trans = pmSourceMatchRelphot(matches, zp, tol, iter1, starRej1, starSys1,
+
+      psVector *trans = pmSourceMatchRelphot(matches, matchedSources, zp, tol, iter1, starRej1, starSys1,
                                                iter2, starRej2, starSys2, starLimit,
                                                transIter, transRej, transThresh); // Transparencies per image
@@ -324 +337 @@
 
         // m_inst_o - m_inst_i = zp[i] - zpTarget - c1 * airmassTarget - 2.5log(sumExpTime) - trans_i
-
+#ifdef TESTING_CZW
+    dumpMatches("source_match2.dat", num, matches, zp, trans);
+#endif
         if (options->matchZPs) {
             options->norm = psVectorAlloc(num, PS_TYPE_F32);
             for (int i = 0; i < num; i++) {
-                if (inputMask->data.U8[i] || !isfinite(trans->data.F32[i])) {
-                    continue;
-                }
+              if (!isfinite(trans->data.F32[i])) {
+                continue;
+              }
                 psArray *sources = sourceLists->data[i]; // Sources of interest
                 float magCorr = zp->data.F32[i] - trans->data.F32[i] - 2.5*log10(options->sumExposure) - airmassTerm * airmassTarget;
@@ -341 +356 @@
                 options->norm->data.F32[i] = magCorr;
                 psLogMsg("ppStack", PS_LOG_INFO,
-                         "Applying scale correction to image %d: %f mag (%f)\n",
-                         i, magCorr, trans->data.F32[i]);
+                         "Applying scale correction to image %d: %f mag (%f) (with inputMask value %d)\n",
+                         i, magCorr, trans->data.F32[i],inputMask->data.U8[i]);
 
                 for (int j = 0; j < sources->n; j++) {
                     pmSource *source = sources->data[j]; // Source of interest
+                    source->psfMag += magCorr;
+                    source->apMag += magCorr;
+
                     if (!source) {
                         continue;
                     }
                     // XXX need to apply to apMag as well
-                    source->psfMag += magCorr;
-                    source->apMag += magCorr;
+                    psTrace("ppStack",5,"Source corrections: %d %d %f %f %f %f %f %f %f %f\n",i,j,
+                            source->peak->xf,source->peak->yf,
+                            source->psfMag,source->psfMagErr,
+                            airmassTerm,airmassTarget,2.5 * log10(options->sumExposure),
+                            source->psfMag + magCorr);
                 }
             }
         }
+        matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches
 
         if (zpExpNum == numGoodImages) {
@@ -366 +388 @@
 
 
-#ifdef TESTING
-        dumpMatches("source_mags.dat", num, matches, zp, trans);
-#endif
         psFree(trans);
 
-#ifdef TESTING
+#ifdef TESTING_CZW
         // Double check: all transparencies should be zero
         {
             psArray *matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches
+            psArray *matchedSources = psArrayAlloc(matches->n);
             if (!matches) {
                 psError(PPSTACK_ERR_DATA, false, "Unable to match sources");
@@ -380 +400 @@
                 return false;
             }
-            psVector *trans = pmSourceMatchRelphot(matches, zp, tol, iter1, starRej1, starSys1,
+            psVector *trans = pmSourceMatchRelphot(matches, matchedSources, zp, tol, iter1, starRej1, starSys1,
                                                    iter2, starRej2, starSys2, starLimit,
                                                    transIter, transRej, transThresh); // Transparencies
@@ -386 +406 @@
                 fprintf(stderr, "Transparency of image %d: %f\n", i, trans->data.F32[i]);
             }
+
             psFree(trans);
             psFree(matches);
-        }
-#endif
-    }
-
-    psFree(zp);
+            psFree(matchedSources);
+        }
+#endif
+    }
+
+
     psFree(zpExp);
 
 #ifdef ASTROMETRY
+    // CZW: This is off by default. 
     // Position offsets
     {
@@ -436 +459 @@
 
 #if (defined TESTING && defined ASTROMETRY)
+    // CZW: This is off by default.
         // Double check: all offsets should be zero
         {
@@ -458 +482 @@
     if (options->photometry) {
         // Save best matches for future photometry
-        psArray *sourcesBest = options->sources = psArrayAllocEmpty(matches->n);
-
+      psArray *sourcesBest = options->sources = psArrayAllocEmpty(matches->n);
         // XXX something of a hack: require at least 2 detections or the nominated fraction of the max possible
         //int minMatches = PS_MAX(2, fracMatch * num);// Minimum number of matches required
         //MEH - if many poor images, will hit oddity with num fraction. use numGoodImages instead
-        int minMatches = PS_MAX(2, fracMatch * numGoodImages);// Minimum number of matches required
-
-        int good0 = 0;
-        for (int i = 0; i < num; i++) {
-            if (inputMask->data.U8[i]) {
-                continue;
-            }
-            good0 = i;
-            break;
+      int minMatches = PS_MAX(2, fracMatch * numGoodImages);// Minimum number of matches required
+      
+      int good0 = 0;
+      for (int i = 0; i < num; i++) {
+        if (inputMask->data.U8[i]) {
+          continue;
         }
-
-        for (int i = 0; i < matches->n; i++) {
-            pmSourceMatch *match = matches->data[i]; // Match of interest
-            if (match->num < minMatches) {
-                continue;
-            }
-
-            // We need to grab a single instance of this source: just take the first available
-            // MEH - if first available is a rejected image, then problem. so take first non-rejected instance above
-            //int image = match->image->data.S32[0]; // Index of image
-            //int index = match->index->data.S32[0]; // Index of source within image
-            int image = match->image->data.S32[good0]; // Index of image
-            int index = match->index->data.S32[good0]; // Index of source within image
-            psArray *sources = sourceLists->data[image]; // Sources for image
-            pmSource *source = sources->data[index]; // Source of interest
-
-            psArrayAdd(sourcesBest, sourcesBest->n, source);
-        }
-        psLogMsg("ppStack", PS_LOG_INFO, "Selected %ld sources for photometry analysis", sourcesBest->n);
-    }
-
+        good0 = i;
+        break;
+      }
+      
+      for (int i = 0; i < matches->n; i++) {
+        pmSourceMatch *match = matches->data[i]; // Match of interest
+         if (match->num < minMatches) {
+           continue;
+         }
+
+         // We need to grab a single instance of this source: just take the first available
+         // MEH - if first available is a rejected image, then problem. so take first non-rejected instance above
+         //      int image = match->image->data.S32[0]; // Index of image
+         // int index = match->index->data.S32[0]; // Index of source within image
+         int image = match->image->data.S32[good0]; // Index of image
+         int index = match->index->data.S32[good0]; // Index of source within image
+         psArray *sources = sourceLists->data[image]; // Sources for image
+         pmSource *source = sources->data[index]; // Source of interest
+
+         // CZW: Accept the first source for all properties, except for the magnitudes.
+         //      We have average magnitudes from relphot, so we should use those
+         pmSource *photsource = matchedSources->data[i];
+         if ((photsource)&&(isfinite(photsource->psfMag))) {
+           // CZW: photsource is the average relphot mag, which is equal to <m_i + zpIN_j>
+           //      (primed are averaged values for this discussion)
+           //      Therefore, the correct magnitude for psfMag' is:
+           //         psfMag' = (photsource - zpIN_j + magCorr)
+           //      apMag - psfMag should be retained, so:
+           //         apMag - psfMag = apMag' - psfMag'
+           //         apMag' = apMag + psfMag' - psfMag
+           source->apMag  = source->apMag + (photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image] - source->psfMag);
+           source->psfMag = photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image];
+           
+           psTrace("ppStack", 5, "Best Source %d/%ld @ %f %f %f %f %f %d",i,sourcesBest->n,source->peak->xf,source->peak->yf,
+                   source->psfMag,
+                   photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image] ,
+                   source->psfMagErr,-1);
+           psArrayAdd(sourcesBest, sourcesBest->n, source);
+         }
+        }
+      psLogMsg("ppStack", PS_LOG_INFO, "Selected %ld sources for photometry analysis", sourcesBest->n);
+
+    }
+    psFree(zp);
     psFree(matches);
 

branches/eam_branches/ipp-20130307/ppStack/src/ppStackTarget.c

@@ -60 +60 @@
         }
         float xSource, ySource;         // Coordinates of source
-        coordsFromSource(&xSource, &ySource, NULL, source);
+        float Smag;                      // magnitude of source
+        coordsFromSource(&xSource, &ySource, &Smag, source);
 
         coords->data.F64[0] = xSource;
@@ -66 +67 @@
 
         psVector *indices = psTreeAllWithin(tree, coords, exclusion); // Number within exclusion zone
-        psTrace("ppStack", 9, "Source at %.0lf,%.0lf has %ld sources in exclusion zone",
-                coords->data.F64[0], coords->data.F64[1], indices->n);
+        psTrace("ppStack", 5, "Filtering: Source at %.0lf %.0lf with mag %f has %ld sources in exclusion zone",
+                coords->data.F64[0], coords->data.F64[1], Smag, indices->n);
         if (indices->n == 1) {
             // Only itself inside the exclusion zone

branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceIO.c

@@ -980 +980 @@
             return false;
         }
+
         // if this is not TRUE, the output files only contain the psf measurements.
         bool XSRC_OUTPUT = psMetadataLookupBool(&status, recipe, "EXTENDED_SOURCE_ANALYSIS");
@@ -1005 +1006 @@
             hdu->header = psFitsReadHeader (NULL, file->fits);
         }
+
+        // "WCS" is CMF without detected objects
+        if (file->type == PM_FPA_FILE_WCS) {
+          psTrace("psModules.objects", 6, "read CMF table from %s : %s : %s", file->filename, headname, dataname);
+          psFree (headname);
+          psFree (dataname);
+          psFree (deteffname);
+          break;
+        }
+
+        // EXTDATA is the PSF data associated with this image header
 
         // we need to find the corresponding table EXTNAME.

branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceMatch.c

@@ -630 +630 @@
 
 psVector *pmSourceMatchRelphot(const psArray *matches, // Array of matches
+                               psArray *matchedSources, // Array of averaged sources
                                const psVector *zp, // Zero points for each image (including airmass term)
                                float tol, // Relative tolerance for convergence
@@ -648 +649 @@
     PS_ASSERT_VECTOR_TYPE(zp, PS_TYPE_F32, NULL);
     PS_ASSERT_FLOAT_LARGER_THAN(transClip, 0.0, NULL);
-
+    PS_ASSERT_ARRAY_NON_NULL(matchedSources, NULL);
+    
     sys1 *= sys1;
     sys2 *= sys2;
@@ -742 +744 @@
     }
 
+    for (int i = 0; i < matches->n; i++) {
+      pmSource *source = pmSourceAlloc();
+      source->psfMag = stars->data.F32[i];
+      psArrayAdd(matchedSources,matchedSources->n,source);
+    }
+    
     psFree(photo);
     psFree(badImage);

branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceMatch.h

@@ -74 +74 @@
 /// Perform relative photometry to calibrate images
 psVector *pmSourceMatchRelphot(const psArray *matches, // Array of matches
+                               psArray *matchedSources, // Array of average sources
                                const psVector *zp, // Zero points for each image (including airmass term)
                                float tol, // Relative tolerance for convergence

branches/eam_branches/ipp-20130307/pswarp/doc/notes.20130406.txt

@@ -31 +31 @@
                         * sets file->fileLevel based on format, but slightly out of sync wrt pmFPAfileDefineFromArgs
                           (NOTE: does not expect / examine any other file)
-                          
 
+
+in pswarpLoop (major blocks):
+
+   * find the needed recipe / filerules / etc
+   * pswarpIOChecksBefore() : generate the outputs?  (isn't this done in pswarpDefine?)
+   * read input astrometry headers (also loads sources or not?)
+   * apply astrometry headers to input->fpa
+   * pswarpTransformReadout : transform pixels, etc
+   * pswarpPixelsLit : measure filling & masks stats
+   * modify the covariance
+   * modify the ZP & trimsec (seems like these should be encapsulated)
+   * update headers
+   * measure the PSF    
+   * measure pixel stats on output
+   * add MD5 to header
+   
+   
+
+----
+
+pswarp issues:
+
+1) operations need to loop over the output astrometry / targets for fpa/chip/cell levels
+   a) I replaced pswarpDefineSkycell with an update to pmFPAfileDefineXX which does the looping
+   b) is it identical in behavior to pswarpDefineSkycell?
+      * YES : I can drop pswarpDefineSkycell 
+
+2) how do we load the SOURCES?
+
+   * I need to load the headers for the astrometry, and I do this by
+     setting the type of the input astrometry to be WCS.  is this the
+     same as CMF, but the latter reads the sources??
+
+   * by changing the filerules entry CMF <-> WCS pswarpDefine.c either does or does not read the sources
+
+     (for testing, I want to use a skycell image or template to define the astrometry; this would not have sources to be read)
+
+3) does the I/O loop in pswarpLoop (pswarpIOChecksBefore) do anything useful?
+   ** I don't think so... this operation is done in pswarpDefine.
+
+4) how do I determine the overlaps correctly? (I'd like to do this before I load sources or pixels)
+
+   ** pswarpMapSetLocalModel does not seem to depend on astrometry model style (GOOD!)
+   

branches/eam_branches/ipp-20130307/pswarp/src/Makefile.am

@@ -35 +35 @@
         pswarpDefine.c                  \
         pswarpDefineBackground.c        \
+        pswarpDefineSkycell.c   \
         pswarpErrorCodes.c              \
         pswarpMapGrid.c                 \

branches/eam_branches/ipp-20130307/pswarp/src/pswarp.c

@@ -39 +39 @@
     }
 
-    // load the skycell layout information
+    // load the skycell layout information:
+    //   read the output astrometry description
+    //   generate the output pixels
+    //   convert header astrometry to pmFPA astrometry structures
     if (!pswarpDefine(config)) {
         pswarpCleanup(config, statsFile);

branches/eam_branches/ipp-20130307/pswarp/src/pswarpDefine.c

@@ -38 +38 @@
     }
     
-    // open the full skycell file; no need to defer different depths. only load the header data
-    pmFPAview *view = pmFPAviewAlloc (0);
-    pmFPAfileOpen (skycell, view, config);
+    // Read the output astrometry
+    // XXX this section loops over the input skycell to load the header info
+    {
+        pmFPAview *view = pmFPAviewAlloc(0);
 
-    // Read header and create target
-    {
-        if (!pmFPAReadHeaderSet(skycell->fpa, skycell->fits, config)) {
-            psError(psErrorCodeLast(), false, "Unable to read headers for skycell.");
-            psFree(view);
-            return false;
-        }
-        view->chip = 0;
-        view->cell = 0;
-        view->readout = 0;
-        pmCell *source = pmFPAfileThisCell(config->files, view, "PSWARP.SKYCELL"); ///< Source cell
-        pmHDU *hdu = pmHDUFromCell(source); ///< HDU for source
-        if (!hdu || !hdu->header) {
-            psError(PM_ERR_PROG, false, "Unable to find header for sky cell.");
-            psFree(view);
-            return false;
-        }
-        int numCols = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); ///< Number of columns
-        int numRows = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); ///< Number of rows
-        if ((numCols == 0) || (numRows == 0)) {
-            psError(PSWARP_ERR_DATA, false, "skycell has invalid dimensions %d x %d", numCols, numRows);
-            psFree(view);
-            return false;
-        }
+        pmChip *chip;
+        if (!pmFPAfileRead (skycell, view, config)) {
+            psError(PS_ERR_IO, false, "failed READ at FPA %s", skycell->name);
+            psFree(view);
+            return false;
+        }
+        while ((chip = pmFPAviewNextChip (view, skycell->fpa, 1)) != NULL) {
+            psTrace ("pswarp", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
+            if (!chip->process || !chip->file_exists) { continue; }
+            if (!pmFPAfileRead (skycell, view, config)) {
+                psError(psErrorCodeLast(), false, "failed READ at CHIP %s", skycell->name);
+                psFree(view);
+                return false;
+            }
+            pmCell *cell;
+            while ((cell = pmFPAviewNextCell (view, skycell->fpa, 1)) != NULL) {
+                psTrace ("pswarp", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+                if (!cell->process || !cell->file_exists) { continue; }
+                if (!pmFPAfileRead (skycell, view, config)) {
+                    psError(psErrorCodeLast(), false, "failed READ at CELL %s", skycell->name);
+                    psFree(view);
+                    return false;
+                }
+                if (!pmFPAfileClose(skycell, view)) {
+                    psError(psErrorCodeLast(), false, "failed CLOSE at CELL %s", skycell->name);
+                    psFree (view);
+                    return false;
+                }
 
-        pmCell *target = pmFPAviewThisCell(view, output->fpa); ///< Target cell
-        pmReadout *readout = pmReadoutAlloc(target); ///< Target readout
-        readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32);
-        psImageInit(readout->image, NAN);
-        psFree(readout);                // Drop reference
-
-        bool status = false;
-        psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.XBIN");
-        psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.YBIN");
-        psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.XSIZE");
-        psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.YSIZE");
-        psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.XPARITY");
-        psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.YPARITY");
-        psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.X0");
-        psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.Y0");
+                // we've got the output astrom header
+                pmHDU *hdu = pmHDUFromCell(cell); ///< HDU for source
+                if (!hdu || !hdu->header) {
+                    psError(PM_ERR_PROG, false, "Unable to find header for sky cell.");
+                    psFree(view);
+                    return false;
+                }
+                int numCols = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); ///< Number of columns
+                int numRows = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); ///< Number of rows
+                if ((numCols == 0) || (numRows == 0)) {
+                    psError(PSWARP_ERR_DATA, false, "skycell has invalid dimensions %d x %d", numCols, numRows);
+                    psFree(view);
+                    return false;
+                }
+                
+                // XXX generate the output pixels
+                pmCell *target = pmFPAviewThisCell(view, output->fpa); ///< Target cell
+                pmReadout *readout = pmReadoutAlloc(target); ///< Target readout
+                readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32);
+                psImageInit(readout->image, NAN);
+                psFree(readout);                // Drop reference
+                
+                bool status = false;
+                psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.XBIN");
+                psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.YBIN");
+                psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.XSIZE");
+                psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.YSIZE");
+                psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.XPARITY");
+                psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.YPARITY");
+                psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.X0");
+                psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.Y0");
+            }
+            if (!pmFPAfileClose(skycell, view)) {
+                psError(psErrorCodeLast(), false, "failed CLOSE at CHIP %s", skycell->name);
+                psFree (view);
+                return false;
+            }
+        }
+        if (!pmFPAfileClose(skycell, view)) {
+            psError(psErrorCodeLast(), false, "failed CLOSE at FPA %s", skycell->name);
+            psFree (view);
+            return false;
+        }
+        psFree(view);
     }
 
-    // XXX this is not a sufficient test
+    // the section below converts the header astrometry info (in skycell) to the pmFPA
+    // astrometry structures, saving them on the output fpa structure.
+
+    // XXX this block should be modified to loop over all chips
+
+    pmFPAview *view = pmFPAviewAlloc (0);
     view->chip = 0;
     view->cell = 0;

branches/eam_branches/ipp-20130307/pswarp/src/pswarpDefineSkycell.c

@@ -73 +73 @@
     psFitsClose(fits);
 
-    // We need to force the format for the skycell to be equivalent to SIMPLE.  Determine
-    // the current format from the header; Determine camera if not specified already
-    // XXX EAM : this operation should be defined as a pmConfig function (pmConfigCopy?)
-    skyConfig = pmConfigAlloc();
-    skyConfig->user = psMemIncrRefCounter(config->user);
-    skyConfig->system = psMemIncrRefCounter(config->system);
+    { // this bit is unique to pswarpDefineSkycell:
 
-    psFree (skyConfig->files);
-    skyConfig->files = psMemIncrRefCounter(config->files);
-    psFree (skyConfig->arguments);
-    skyConfig->arguments = psMemIncrRefCounter(config->arguments);
+      // We need to force the format for the skycell to be equivalent to SIMPLE.  Determine
+      // the current format from the header; Determine camera if not specified already
+      // XXX EAM : this operation should be defined as a pmConfig function (pmConfigCopy?)
+      skyConfig = pmConfigAlloc();
+      skyConfig->user = psMemIncrRefCounter(config->user);
+      skyConfig->system = psMemIncrRefCounter(config->system);
+
+      psFree (skyConfig->files);
+      skyConfig->files = psMemIncrRefCounter(config->files);
+      psFree (skyConfig->arguments);
+      skyConfig->arguments = psMemIncrRefCounter(config->arguments);
+    }
 
     format = pmConfigCameraFormatFromHeader (NULL, NULL, NULL, skyConfig, phu, false);
@@ -94 +97 @@
 
     // Record the camera name of the skycell, so we can save its configuration
-    psMetadataAddStr(config->arguments, PS_LIST_TAIL, "SKYCELL.CAMERA", 0, "Name of camera for skycell",
-                     skyConfig->cameraName);
+    // XXX this can be put outside of the f
+    psMetadataAddStr(config->arguments, PS_LIST_TAIL, "SKYCELL.CAMERA", 0, "Name of camera for skycell", skyConfig->cameraName);
 
     // build the template fpa, set up the basic view

branches/eam_branches/ipp-20130307/pswarp/src/pswarpLoop.c

@@ -77 +77 @@
     // Turn all skycell files on to generate them, and then turn them off for the loop over the input images
     // the input, which is in a different format.
-    {
-        pswarpFileActivation(config, detectorFiles, false);
-        pswarpFileActivation(config, photFiles, false);
-        pswarpFileActivation(config, independentFiles, false);
-        pswarpFileActivation(config, skycellFiles, true);
-        if (!pswarpIOChecksBefore(config)) {
-            psError(psErrorCodeLast(), false, "Unable to read files.");
-            goto DONE;
-        }
-        pswarpFileActivation(config, skycellFiles, false);
-    }
+    // XXX why is this being done here, doesn't it duplicate the work in pswarpDefine.c??
+    pswarpFileActivation(config, detectorFiles, false);
+    pswarpFileActivation(config, photFiles, false);
+    pswarpFileActivation(config, independentFiles, false);
+    pswarpFileActivation(config, skycellFiles, false);
+    
+    // pswarpFileActivation(config, skycellFiles, true);
+    // if (0) {
+    //   if (!pswarpIOChecksBefore(config)) {
+    //     psError(psErrorCodeLast(), false, "Unable to read files.");
+    //     goto DONE;
+    //   }
+    // }
+   
 
     // Read the input astrometry
@@ -164 +167 @@
     }
 
+    // *** main transformation block
     pmChip *chip;
     while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) {

branches/eam_branches/ipp-20130307/pswarp/src/pswarpParseCamera.c

@@ -55 +55 @@
 }
 
-
-
-
 bool pswarpParseCamera(pmConfig *config)
 {
@@ -70 +67 @@
     }
 
-    pmFPAfile *astrom = defineInputFile(config, NULL, "PSWARP.ASTROM", "ASTROM", PM_FPA_FILE_CMF);
+    // XXX I should allow the astrometry input to be either WCS or CMF
+    // Define the input astrometry file(s)
+    bool status = false;
+    pmFPAfile *astrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM");
+    if (!astrom) {
+        // look for the file on the RUN metadata
+        astrom = pmFPAfileDefineFromRun(&status, NULL, config, "PSWARP.ASTROM"); // File to return
+        if (!status) {
+            psLogMsg("pswarp", PS_LOG_INFO, "Failed to load file definition for %s", "PSWARP.ASTROM");
+        }
+    }
+    if ((astrom->type != PM_FPA_FILE_CMF) && (astrom->type != PM_FPA_FILE_WCS)) {
+        psLogMsg("pswarp", PS_LOG_INFO, "%s is neither CMF nor WCS", "PSWARP.ASTROM");
+    }
     psLogMsg("pswarp", PS_LOG_INFO, "Astrometry source: %s", astrom ? "supplied" : "header");
 
+    // Define the input mask file(s)
     pmFPAfile *inMask = defineInputFile(config, input, "PSWARP.MASK", "MASK", PM_FPA_FILE_MASK);
     if (!inMask) {
@@ -78 +89 @@
     }
 
+    // Define the input variance file(s)
     pmFPAfile *inVariance = defineInputFile(config, input, "PSWARP.VARIANCE", "VARIANCE",
                                             PM_FPA_FILE_VARIANCE);
@@ -91 +103 @@
     
     // The input skycell is a required argument: it defines the output image
-    bool status = false;
     pmConfig *skyConfig = NULL;
-    pmFPAfile *skycell = pmFPAfileDefineNewConfig(&status, &skyConfig, config, "PSWARP.SKYCELL", "SKYCELL");
+    pmFPAfile *skycell = NULL;
+
+# define USE_PSWARP_DEFINE_SKYCELL 0
+# if (USE_PSWARP_DEFINE_SKYCELL)
+    // XXX this function is no longer needed; drop
+    status = pswarpDefineSkycell(&skycell, &skyConfig, config, "PSWARP.SKYCELL", "SKYCELL");
+# else
+    skycell = pmFPAfileDefineNewConfig(&status, &skyConfig, config, "PSWARP.SKYCELL", "SKYCELL");
+# endif
+
     if (!status || !skycell) {
       psError(psErrorCodeLast(), false, "Failed to build FPA from PSWARP.SKYCELL");