Changeset 36480 for branches/eam_branches/ipp-20131211

Timestamp:

Feb 6, 2014, 11:48:53 AM (12 years ago)

Author:

eugene

Message:

merging changes from trunk

Location:

branches/eam_branches/ipp-20131211

Files:

• . (modified) (1 prop)
• Nebulous (modified) (1 prop)
• Nebulous-Server (modified) (1 prop)
• Ohana (modified) (1 prop)
• Ohana/src/dvomerge (modified) (1 prop)
• Ohana/src/libohana/src (modified) (1 prop)
• Ohana/src/opihi (modified) (1 prop)
• Ohana/src/opihi/cmd.astro (modified) (1 prop)
• Ohana/src/opihi/cmd.data (modified) (1 prop)
• Ohana/src/relastro/src (modified) (1 prop)
• dbconfig/cam.md (modified) (1 prop)
• dbconfig/changes.txt (modified) (1 diff, 1 prop)
• dbconfig/ff.md (copied) (copied from trunk/dbconfig/ff.md )
• dbconfig/ipp.m4 (modified) (1 diff)
• ippMonitor (modified) (1 prop)
• ippMonitor/czartool/czartool/Gpc1Db.pm (modified) (6 diffs)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/chip_imfile.pl (modified) (1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/dist_bundle.pl (modified) (6 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/lap_science.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/psphot_fullforce_summary.pl (copied) (copied from trunk/ippScripts/scripts/psphot_fullforce_summary.pl )
• ippScripts/scripts/psphot_fullforce_warp.pl (modified) (18 diffs)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/queuestaticsky.pl (modified) (2 diffs)
• ippScripts/scripts/stack_skycell.pl (modified) (4 diffs)
• ippScripts/scripts/staticsky.pl (modified) (4 diffs)
• ippScripts/scripts/warp_skycell.pl (modified) (1 prop)
• ippTasks (modified) (1 prop)
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/fullforce.pro (copied) (copied from trunk/ippTasks/fullforce.pro )
• ippTasks/pstamp.pro (modified) (2 diffs)
• ippTasks/rawcheck.pro (modified) (1 diff)
• ippToPsps (modified) (1 prop)
• ippToPsps/jython/dvo.py (modified) (2 diffs)
• ippToPsps/jython/gpc1db.py (modified) (2 diffs)
• ippToPsps/jython/queue.py (modified) (1 diff)
• ippToPsps/jython/stackbatch.py (modified) (2 diffs)
• ippTools/share (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_pendingcleanuprun.sql (modified) (1 diff)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/fftool_definebyquery.sql (copied) (copied from trunk/ippTools/share/fftool_definebyquery.sql )
• ippTools/share/fftool_definebyquery_select_warps.sql (copied) (copied from trunk/ippTools/share/fftool_definebyquery_select_warps.sql )
• ippTools/share/fftool_result.sql (copied) (copied from trunk/ippTools/share/fftool_result.sql )
• ippTools/share/fftool_revert.sql (copied) (copied from trunk/ippTools/share/fftool_revert.sql )
• ippTools/share/fftool_revertsummary.sql (copied) (copied from trunk/ippTools/share/fftool_revertsummary.sql )
• ippTools/share/fftool_summary.sql (copied) (copied from trunk/ippTools/share/fftool_summary.sql )
• ippTools/share/fftool_summaryinputs.sql (copied) (copied from trunk/ippTools/share/fftool_summaryinputs.sql )
• ippTools/share/fftool_toadvance.sql (copied) (copied from trunk/ippTools/share/fftool_toadvance.sql )
• ippTools/share/fftool_todo.sql (copied) (copied from trunk/ippTools/share/fftool_todo.sql )
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff, 1 prop)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/src (modified) (2 props)
• ippTools/src/Makefile.am (modified) (3 diffs)
• ippTools/src/camtool.c (modified) (1 prop)
• ippTools/src/camtoolConfig.c (modified) (1 prop)
• ippTools/src/chiptool.c (modified) (2 diffs)
• ippTools/src/fftool.c (copied) (copied from trunk/ippTools/src/fftool.c )
• ippTools/src/fftool.h (copied) (copied from trunk/ippTools/src/fftool.h )
• ippTools/src/fftoolConfig.c (copied) (copied from trunk/ippTools/src/fftoolConfig.c )
• ippTools/src/magictool.c (modified) (1 prop)
• ippconfig (modified) (1 prop)
• ippconfig/gpc1 (modified) (1 prop)
• ippconfig/gpc1/psastro.config (modified) (1 diff)
• ippconfig/recipes/filerules-split.mdc (modified) (1 diff)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/psastro.config (modified) (2 diffs)
• ippconfig/recipes/psphot.config (modified) (7 diffs)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 diff, 1 prop)
• ppImage/src (modified) (1 prop)
• ppSub/src/ppSubMatchPSFs.c (modified) (1 diff)
• psLib/src/math/psMixtureModels.c (modified) (4 diffs, 1 prop)
• psModules (modified) (1 prop)
• psModules/src/camera/pmFPAfile.c (modified) (1 diff)
• psModules/src/objects/pmModelClass.c (modified) (3 diffs)
• psModules/src/objects/pmModelClass.h (modified) (1 diff)
• psModules/src/objects/pmSourceIO.c (modified) (7 diffs)
• psModules/src/objects/pmSourceIO.h (modified) (1 diff)
• psModules/src/objects/pmSourceIO_CFF.c (modified) (8 diffs)
• psModules/src/objects/pmSourceIO_CMF.c.in (modified) (9 diffs)
• psModules/src/objects/pmSourceIO_PS1_CAL_0.c (modified) (1 diff)
• psModules/src/objects/pmSourceIO_PS1_DEV_0.c (modified) (1 diff)
• psModules/src/objects/pmSourceIO_PS1_DEV_1.c (modified) (1 diff)
• psModules/src/objects/pmSourceIO_SMPDATA.c (modified) (1 diff)
• psastro/src/psastroDataSave.c (modified) (2 diffs)
• psastro/src/psastroDefineFiles.c (modified) (1 diff)
• psconfig (modified) (1 prop)
• psphot (modified) (1 prop)
• psphot/src (modified) (2 props)
• psphot/src/Makefile.am (modified) (3 diffs)
• psphot/src/psmakecff.c (modified) (8 diffs)
• psphot/src/psphot.h (modified) (1 diff)
• psphot/src/psphotChooseAnalysisOptions.c (modified) (19 diffs)
• psphot/src/psphotFullForceSummary.c (copied) (copied from trunk/psphot/src/psphotFullForceSummary.c )
• psphot/src/psphotFullForceSummaryReadout.c (copied) (copied from trunk/psphot/src/psphotFullForceSummaryReadout.c )
• psphot/src/psphotGalaxyShape.c (modified) (3 diffs)
• psphot/src/psphotStackImageLoop.c (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/dqueryparse.pl (modified) (1 diff)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (4 diffs)
• psvideophot (modified) (1 prop)
• tools/mysql-dump/gpc1_install.sh (modified) (1 diff)
• tools/neb_rawOTA_host_scan.pl (modified) (7 diffs)

branches/eam_branches/ipp-20131211/dbconfig/changes.txt

@@ -2432 +2432 @@
 UPDATE dbversion set schema_version = '1.1.77', updated= CURRENT_TIMESTAMP();
 
+--
+
+CREATE TABLE fullForceRun (
+    ff_id           BIGINT NOT NULL AUTO_INCREMENT,
+    skycal_id       BIGINT,
+    sources_path_base VARCHAR(255),
+    state           VARCHAR(64),
+    workdir         VARCHAR(255),
+    label           VARCHAR(64),
+    data_group      VARCHAR(64),
+    dist_group      VARCHAR(64),
+    note            VARCHAR(255),
+    reduction       VARCHAR(64),
+    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
+    PRIMARY KEY(ff_id),
+    KEY(state),
+    KEY(label),
+    KEY(data_group),
+    KEY(skycal_id),
+    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceInput (
+    ff_id           BIGINT,
+    warp_id         BIGINT,
+    PRIMARY KEY(ff_id, warp_id),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceResult (
+    ff_id           BIGINT,
+    warp_id         BIGINT,
+    path_base       VARCHAR(255) NOT NULL,
+    dtime_script    FLOAT,
+    quality         SMALLINT NOT NULL,
+    hostname        VARCHAR(64) NOT NULL,
+    software_ver    VARCHAR(16),
+    fault           SMALLINT NOT NULL,
+    PRIMARY KEY(ff_id, warp_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceSummary (
+    ff_id           BIGINT,
+    path_base       VARCHAR(255) NOT NULL,
+    dtime_script    FLOAT,
+    quality         SMALLINT NOT NULL,
+    hostname        VARCHAR(64) NOT NULL,
+    software_ver    VARCHAR(16),
+    fault           SMALLINT NOT NULL,
+    PRIMARY KEY(ff_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+UPDATE dbversion set schema_version = '1.1.78', updated= CURRENT_TIMESTAMP();
+

branches/eam_branches/ipp-20131211/dbconfig/ipp.m4

@@ -40 +40 @@
 include(skycell.md)
 include(release.md)
+include(ff.md)

branches/eam_branches/ipp-20131211/ippMonitor/czartool/czartool/Gpc1Db.pm

@@ -139 +139 @@
     my $stateCol =  $table.".state";
 
+    my $extraState = "";
+    if ($state eq 'new') {
+        $extraState = " OR $stateCol = 'update' ";
+    }
+
     my $query =  $self->{_db}->prepare(<<SQL);
     SELECT COUNT(DISTINCT $id) 
@@ -144 +149 @@
         JOIN $joinTable USING ($id)
         WHERE label LIKE '$label'
-        AND (($faultCol != 0 AND $stateCol = '$state') OR $stateCol = 'failed_revert')
+        AND ($faultCol != 0 AND ($stateCol = '$state' $extraState))
 SQL
     $query->execute;
@@ -208 +213 @@
     my $query = undef;
 
+    my $joinTable = undef;
+    my $id = undef;
+    if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;}
+
+    # XXX: the "update" page is now obsolete. The standard processing page should now show
+    # counts in new and update state
     if ($state eq "update") {
-
         if ($stage eq "dist") {return 0;}
-        my $joinTable = undef;
-        my $id = undef;
-        if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;}
 
         if ($stage eq "chip" || $stage eq "fake" || $stage eq "warp" || $stage eq "diff" || $stage eq "magicDS") {
@@ -226 +233 @@
         }
         else {
-        
             $query = $self->{_db}->prepare(<<SQL);
             SELECT COUNT(state)
-                FROM $table JOIN $joinTable 
-                USING($id) 
+                FROM $table
                 WHERE label LIKE '$label'
                 AND state = 'update'; 
 SQL
+        
         }
 
@@ -240 +246 @@
     elsif ($state eq "new") {
 
-        $query = $self->{_db}->prepare(<<SQL);
-        SELECT COUNT(state)  
-            FROM $table 
-            WHERE label LIKE '$label' 
-            AND (state = 'new' OR state = 'update')
-SQL
+        if ($stage eq "chip" || $stage eq "fake" || $stage eq "warp" || $stage eq "diff" || $stage eq "magicDS") {
+            # get count of runs in new state and update state that have components left to update
+            $query = $self->{_db}->prepare(<<SQL);
+            SELECT COUNT(DISTINCT $id)  
+                FROM $table LEFT JOIN $joinTable
+                USING($id)
+                WHERE label LIKE '$label' 
+                AND (state = 'new' OR (state = 'update' AND data_state = 'update'))
+SQL
+        } else {
+            $query = $self->{_db}->prepare(<<SQL);
+            SELECT COUNT(state)
+                FROM $table
+                WHERE label LIKE '$label'
+                AND (state = 'new' OR state = 'update'); 
+SQL
+        }
     }
     else {
@@ -256 +273 @@
 SQL
     }
+
+    # print $query->{Statement} . "\n";
 
     $query->execute;

branches/eam_branches/ipp-20131211/ippScripts/Build.PL

@@ -133 +133 @@
         scripts/queuestaticsky.pl
         scripts/psphot_fullforce_warp.pl
+        scripts/psphot_fullforce_summary.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',

branches/eam_branches/ipp-20131211/ippScripts/scripts/dist_bundle.pl

@@ -50 +50 @@
                      'PPSUB.INVERSE.MASK' => 'inv_mask',
                      'PPSUB.INVERSE.VARIANCE' => 'inv_variance' );
-my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image',
+
+my %stack_cleaned = ( 'PPSTACK.UNCONV' => 'image',
+                      'PPSTACK.UNCONV.MASK' => 'mask',
+                      'PPSTACK.UNCONV.VARIANCE' => 'variance',
+                      'PPSTACK.UNCONV.EXP' => 'exp',
+                      'PPSTACK.UNCONV.EXPNUM' => 'expnum',
+                      'PPSTACK.UNCONV.EXPWT' => 'expwt'
+                      );
+
+my %stack_convolved = ( 'PPSTACK.OUTPUT' => 'image',
                       'PPSTACK.OUTPUT.MASK' => 'mask',
-                      'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
+                      'PPSTACK.OUTPUT.VARIANCE' => 'variance',
+                      'PPSTACK.OUTPUT.EXP' => 'exp',
+                      'PPSTACK.OUTPUT.EXPNUM' => 'expnum',
+                      'PPSTACK.OUTPUT.EXPWT' => 'expwt'
+                      );
 my %empty_cleaned = ();
+
 
 
@@ -136 +150 @@
 if (($stage ne 'raw') and ($stage ne 'fake') and ($stage ne 'stack_summary') and !$poor_quality) {
     # If the file list is empty it is an error because we should at least get a config dump file
-    # except for fake stage which doesn't do anything yet
+    # except for fake stage which doesn't do anything yet, stack_summary stage, and of course raw files have
+    # no config dump because they aren't processed
     &my_die("empty file list", $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
 }
@@ -165 +180 @@
     my $image_type = get_image_type($stage, $file_rule);
 
+    # skip useless empty trace files
+    next if ($file_rule =~ /TRACE/);
+
     # if this is an image and we are building a clean bundle or if quality is bad skip this rule
     next if $image_type and ($clean or $poor_quality);
+
+    if ($stage eq 'stack') {
+        # skip convolved stacks since they are deleted just after they are created now
+        next if $stack_convolved{$file_rule};
+    }
 
     # if magic is required, don't ship jpegs or binned fits images
@@ -180 +203 @@
 
     my $file_name = $file->{name};
-    my $base = basename($file_name);
     my $path = $ipprc->file_resolve($file_name);
+
+    if (!$path and $file_rule =~ /LOG/) {
+        my $compressed_file_name = $file_name . '.bz2';
+        my $compressed_path = $ipprc->file_resolve($compressed_file_name);
+        if ($compressed_path) {
+            $path = $compressed_path;
+            $file_name = $compressed_file_name;
+        }
+    }
+
 
     if (!$path) {
@@ -194 +226 @@
         # XXX: perhaps only do this for stages where we know that this happens
         next if $file_rule =~ /STATS/;
-        # don't fail on these "non-essential files"
+        # don't fail if these "non-essential files" are missing
+        next if $file_rule =~ /LOG/;
         next if $file_rule =~ /TRACE/;
-        next if $file_rule =~ /LOG/;
         next if $file_rule =~ /BIN/;
 
@@ -236 +268 @@
         # create a symbolic link from the file in the nebulous repository
         # in the temporary directory
-        symlink $path, "$tmpdir/$base";
+        symlink $path, "$tmpdir/" . basename($file_name);
     }
 }

branches/eam_branches/ipp-20131211/ippScripts/scripts/psphot_fullforce_warp.pl

@@ -31 +31 @@
 my $psphotFullForce = can_run('psphotFullForce') or (warn "Can't find psphotFullForce" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
-# XXX: fftool is yet to be written
-my $fftool = "fftool";  # can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1);
+my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -38 +37 @@
 }
 
-my ($ffw_id, $warp_id, $skycell_id, $path_base, $sourceroot, $camera);
+my ($ff_id, $warp_id, $skycell_id, $path_base, $sourceroot, $camera);
 my ($outroot, $reduction);
 my ($dbname, $threads, $verbose, $no_update, $no_op, $redirect);
 
 GetOptions(
-    'ffw_id=s'          => \$ffw_id,
+    'ff_id=s'          => \$ff_id,
     'warp_id=s'         => \$warp_id,   # warp identifier
     'skycell_id=s'      => \$skycell_id,# Skycell identifier
     'warp_path_base=s'  => \$path_base, # path_base of the warp skycell
-    'sourceroot=s'      => \$sourceroot,# path_base of sources
+    'sources_path_base=s' => \$sourceroot,# path_base of sources
     'camera=s'          => \$camera,    # camera name of sources
     'dbname|d=s'        => \$dbname,    # Database name
@@ -61 +60 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-    -msg => "Required options: --ffw_id --sourceroot --skycell_id --warp_path_base --outroot --camera",
+    -msg => "Required options: --ff_id --warp_id --sourceroot --skycell_id --warp_path_base --outroot --camera",
     -exitval => 3,
-          ) unless defined $ffw_id,
+          ) unless defined $ff_id,
     and defined $sourceroot
-    and (defined $path_base or defined $warp_id) # if we don't have warp's path_base we need warp_id
+    and defined $path_base 
+    and defined $warp_id
     and defined $skycell_id
     and defined $camera
     and defined $outroot;
 
-# XXX: fftool is not ready to run commands that update the database
-$no_update = 1;
-
-my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $ffw_id, $skycell_id, $PS_EXIT_CONFIG_ERROR );
+my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $ff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR );
 
 my $neb;
@@ -84 +81 @@
 
 $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", 
-    $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
-
-if (!$path_base) {
+    $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
+
+if (0) { 
+# if (!$path_base) {
     # If path_base is not supplied, look it up in the database.
     my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -98 +96 @@
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
             &my_die("Unable to perform warptool -warpskyfile -inputskyfile: $error_code", 
-                $ffw_id, $skycell_id, $error_code);
+                $ff_id, $warp_id, $skycell_id, $error_code);
         }
 
         my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-            &my_die("Unable to parse metadata config doc", $ffw_id, $skycell_id, $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to parse metadata config doc", $ff_id, $warp_id, $skycell_id, $PS_EXIT_PROG_ERROR);
         $files = parse_md_list($metadata) or
-            &my_die("Unable to parse metadata list", $ffw_id, $skycell_id, $PS_EXIT_PROG_ERROR);
-    }
-
-    &my_die("Input list does not contain exactly one elements", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) 
+            &my_die("Unable to parse metadata list", $ff_id, $warp_id, $skycell_id, $PS_EXIT_PROG_ERROR);
+    }
+
+    &my_die("Input list does not contain exactly one elements", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) 
         unless scalar @$files == 1;
 
@@ -113 +111 @@
 
     $path_base = $warp->{path_base};
-    &my_die("Couldn't find input path in warptool output", $ffw_id, $skycell_id, $PS_EXIT_UNKNOWN_ERROR) 
+    &my_die("Couldn't find input path in warptool output", $ff_id, $warp_id, $skycell_id, $PS_EXIT_UNKNOWN_ERROR) 
         unless defined $path_base;
 
@@ -123 +121 @@
 unless ($recipe_psphot) {
     &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", 
-        $ffw_id, $skycell_id, $PS_EXIT_CONFIG_ERROR);
+        $ff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR);
 }
 
@@ -134 +132 @@
 print "recipe_psphot: $recipe_psphot\n";
 
+# use psf measured on input warp
+# XXX: get this from recipe
+my $useWarpPSF = 0;
+
 my $input         = $ipprc->filename('PSWARP.OUTPUT', $path_base);
 my $inputMask     = $ipprc->filename('PSWARP.OUTPUT.MASK', $path_base);
 my $inputVariance = $ipprc->filename('PSWARP.OUTPUT.VARIANCE', $path_base);
-my $inputPSF      = $ipprc->filename('PSPHOT.PSF.SKY.SAVE', $path_base);
+my $inputPSF      = $useWarpPSF ? $ipprc->filename('PSPHOT.PSF.SKY.SAVE', $path_base) : "";
 my $inputSources  = $ipprc->filename('PSPHOT.OUTPUT.CFF', $sourceroot);
 
@@ -144 +146 @@
     print "inputMask:     $inputMask\n";
     print "inputVariance: $inputVariance\n";
-    # print "inputPath:     $path_base\n";
-    print "inputPSF:      $inputPSF\n";
+    print "inputPSF:      $inputPSF\n" if $inputPSF;
     print "inputSources:  $inputSources\n";
 }
 
 # check that the inputs exist (and have non-zero size)
-&my_die("Couldn't find input: $input", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input);
-&my_die("Couldn't find input: $inputMask", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask);
-&my_die("Couldn't find input: $inputVariance", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance);
-&my_die("Couldn't find input: $inputPSF", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputPSF);
-&my_die("Couldn't find input: $inputSources", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputSources);
+&my_die("Couldn't find input: $input", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input);
+&my_die("Couldn't find input: $inputMask", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask);
+&my_die("Couldn't find input: $inputVariance", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance);
+&my_die("Couldn't find input: $inputPSF", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) if ($inputPSF && !$ipprc->file_exists($inputPSF));
+&my_die("Couldn't find input: $inputSources", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputSources);
 
 my $dump_config = 1; 
@@ -193 +194 @@
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform ppSub: $error_code", $ffw_id, $skycell_id, $error_code);
+            &my_die("Unable to perform ppSub: $error_code", $ff_id, $warp_id, $skycell_id, $error_code);
         }
 
@@ -207 +208 @@
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform ppStatsFromMetadata: $error_code", $ffw_id, $skycell_id, $error_code);
+                &my_die("Unable to perform ppStatsFromMetadata: $error_code", $ff_id, $warp_id, $skycell_id, $error_code);
             }
             foreach my $line (@$stdout_buf) {
@@ -226 +227 @@
 # Add the result to the database
 {
-    my $command = "$fftool -ffw_id $ffw_id -skycell_id $skycell_id";
-    $command .= " -addwarped -path_base $outroot";
+    my $command = "$fftool -ff_id $ff_id -warp_id $warp_id"; 
+    $command .= " -addresult -path_base $outroot";
     $command .= " $cmdflags";
     $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
@@ -239 +240 @@
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
             my $err_message = "Unable to perform fftool -addwarped" ;
-                &my_die("$err_message: $error_code", $ffw_id, $skycell_id, $error_code);
+                &my_die("$err_message: $error_code", $ff_id, $warp_id, $skycell_id, $error_code);
         }
     } else {
@@ -262 +263 @@
     my $error;
     my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error)
-                    or &my_die("failed to prepare output file for: $filerule", $ffw_id, $skycell_id, $error);
+                    or &my_die("failed to prepare output file for: $filerule", $ff_id, $warp_id, $skycell_id, $error);
     return $output;
 }
@@ -275 +276 @@
     }
 
-    &my_die("Couldn't find expected output file: $file",  $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
+    &my_die("Couldn't find expected output file: $file",  $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
 
     # Funpack to confirm we've really made things correctly
     my $diskfile = $ipprc->file_resolve($file);
     if ($diskfile =~ /fits/) {
-        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR);
         my $check_command = "$funpack -S $diskfile > /dev/null";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $check_command, verbose => $verbose);
         if (!$success) {
-            &my_die("Output file not a valid fits file: $file", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+            &my_die("Output file not a valid fits file: $file", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR);
         }
     }
@@ -291 +292 @@
 
     if ($replicate and $neb) {
-        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR);
     }
 }
@@ -299 +300 @@
 {
     my $msg = shift;            # Warning message on die
-    my $ffw_id = shift;         # full force warp identifier
+    my $ff_id = shift;          # full force run identifier
+    my $warp_id = shift;        # full force warp id
     my $skycell_id = shift;     # Skycell identifier
     my $exit_code = shift;      # Exit code to add
@@ -306 +308 @@
 
     warn($msg);
-    if (defined $ffw_id and defined $skycell_id) {
-        my $command = "$fftool -ffw_id $ffw_id -skycell_id $skycell_id -fault $exit_code";
-        $command .= " -addffskyfile";
+    if (defined $ff_id and defined $skycell_id) {
+        my $command = "$fftool -ff_id $ff_id -warp_id $warp_id -fault $exit_code";
+        $command .= " -addresult";
         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
         $command .= " -hostname $host" if defined $host;

branches/eam_branches/ipp-20131211/ippScripts/scripts/queuestaticsky.pl

@@ -33 +33 @@
 };
 
+# don't update the database if we are pretending
+$no_update = 1 if $pretend;
 
 my $missing_tools;
@@ -113 +115 @@
         }
     }
+    last if $pretend;   # only do the first (full) set if we are in pretend mode
 }
 {

branches/eam_branches/ipp-20131211/ippScripts/scripts/stack_skycell.pl

@@ -40 +40 @@
 }
 
-my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps);
+my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps, $delete_convolved_images);
 GetOptions(
     'stack_id|d=s'      => \$stack_id, # Stack identifier
@@ -55 +55 @@
     'redirect-output'   => \$redirect,
     'save-temps'        => \$save_temps, # Save temporary files?
+    'delete-convolved'  => \$delete_convolved_images,
 ) or pod2usage( 2 );
 
@@ -92 +93 @@
 );
 
+
 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 $| = 1;
@@ -360 +362 @@
 
     if (!$quality) {
+        
+
         check_outputs(\@outputFiles, $replicate_outputs);
+
+        if (($convolve)&&($delete_convolved_images)) { # We made convolved products, but do not wish to keep them anymore
+            my @products_to_clear = ('PPSTACK.OUTPUT',
+                                     'PPSTACK.OUTPUT.MASK',
+                                     'PPSTACK.OUTPUT.VARIANCE',
+                                     'PPSTACK.OUTPUT.EXP',
+                                     'PPSTACK.OUTPUT.EXPNUM',
+                                     'PPSTACK.OUTPUT.EXPWT');
+            foreach my $product (@products_to_clear) {
+                my $file = $ipprc->filename($product,$outroot,$skycell_id);
+                print "Deleting unwanted convolved product: $file\n";
+                my $error= $ipprc->kill_file($file);
+                if ($error_code) {
+                    print "Failed to delete unwanted convolved product: $error_code\n";
+                }
+            }
+        }
+
     }
 

branches/eam_branches/ipp-20131211/ippScripts/scripts/staticsky.pl

@@ -138 +138 @@
         my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot);
 
+        my $needConvolvedImages = 0;
+
         foreach my $file (@$files) {
             print $listFile "INPUT   METADATA\n";
@@ -146 +148 @@
             my $stack_id = $file->{stack_id};
 
-            my $imageCnv  = $ipprc->filename("PPSTACK.OUTPUT",          $path_base ); # Image name
-            my $maskCnv   = $ipprc->filename("PPSTACK.OUTPUT.MASK",     $path_base ); # Mask name
-            my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name
-            my $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM",   $path_base ); # Expnum name
+            my ($imageCnv, $maskCnv, $weightCnv, $expnumCnv);
+            if ($needConvolvedImages) {
+                $imageCnv  = $ipprc->filename("PPSTACK.OUTPUT",          $path_base ); # Image name
+                $maskCnv   = $ipprc->filename("PPSTACK.OUTPUT.MASK",     $path_base ); # Mask name
+                $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name
+                $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM",   $path_base ); # Expnum name
+            }
 
             my $imageRaw  = $ipprc->filename("PPSTACK.UNCONV",          $path_base ); # Image name
@@ -166 +171 @@
             &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw");
             &my_die("Couldn't find input: $expnumRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumRaw");
-            &my_die("Couldn't find input: $imageCnv",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv");
-            &my_die("Couldn't find input: $maskCnv",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv");
-            &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv");
-            &my_die("Couldn't find input: $expnumCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv");
-            &my_die("Couldn't find input: $psfCnv",    $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv");
+            if ($needConvolvedImages) {
+                &my_die("Couldn't find input: $imageCnv",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv");
+                &my_die("Couldn't find input: $maskCnv",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv");
+                &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv");
+                &my_die("Couldn't find input: $expnumCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv");
+                &my_die("Couldn't find input: $psfCnv",    $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv");
+            }
             &my_die("Couldn't find input: $sources",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources");
 
@@ -178 +185 @@
             print $listFile "  RAW:VARIANCE  STR  " . $weightRaw . "\n";
             print $listFile "  RAW:EXPNUM    STR  " . $expnumRaw . "\n";
-
-            print $listFile "  CNV:IMAGE     STR  " . $imageCnv  . "\n";
-            print $listFile "  CNV:MASK      STR  " . $maskCnv   . "\n";
-            print $listFile "  CNV:VARIANCE  STR  " . $weightCnv . "\n";
-            print $listFile "  CNV:EXPNUM    STR  " . $expnumCnv . "\n";
-            print $listFile "  CNV:PSF       STR  " . $psfCnv    . "\n";
-
             print $listFile "  SOURCES       STR  " . $sources   . "\n";
+            if ($needConvolvedImages) {
+                print $listFile "  CNV:IMAGE     STR  " . $imageCnv  . "\n";
+                print $listFile "  CNV:MASK      STR  " . $maskCnv   . "\n";
+                print $listFile "  CNV:VARIANCE  STR  " . $weightCnv . "\n";
+                print $listFile "  CNV:EXPNUM    STR  " . $expnumCnv . "\n";
+                print $listFile "  CNV:PSF       STR  " . $psfCnv    . "\n";
+            }
 
             print $listFile "END\n\n";

branches/eam_branches/ipp-20131211/ippTasks/Makefile.am

@@ -46 +46 @@
         diffphot.pro \
         lap.pro \
+        lapgroup.pro \
         vp.pro \
-        bg.regeneration.pro
+        bg.regeneration.pro \
+        fullforce.pro
 
 other_files = \

branches/eam_branches/ipp-20131211/ippTasks/pstamp.pro

@@ -825 +825 @@
         book getword pstampDependent $pageName imagedb    -var IMAGEDB
         book getword pstampDependent $pageName rlabel     -var RLABEL
+        book getword pstampDependent $pageName label      -var LABEL
         book getword pstampDependent $pageName outdir     -var OUTDIR
         book getword pstampDependent $pageName need_magic -var NEED_MAGIC
@@ -846 +847 @@
         stderr $MYLOGFILE
 
-        $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT --logfile $MYLOGFILE
+        $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL --label $LABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT --logfile $MYLOGFILE
 
         add_standard_args run

branches/eam_branches/ipp-20131211/ippTasks/rawcheck.pro

@@ -75 +75 @@
   task.exec
       book npages rawcheckPending -var N
-      if ($N > 2000)
+      if ($N > 200)
         process_cleanup rawcheckPending
         break

branches/eam_branches/ipp-20131211/ippToPsps/jython/dvo.py

@@ -65 +65 @@
         if not self.correctDvo:
             print "*******************************************************************************"
-            self.logger.warning("switching to a new dvo: " + self.skychunk.dvoLocation)
+            self.logger.warning("switching to a new dvo: '" + self.skychunk.dvoLocation + "'")
             # response = raw_input("(y/n) ")
             # if response == "y":
@@ -669 +669 @@
             cmd += " -time-start " + time_start
             cmd += " -time-end " + time_end
-   
+
+        useP2 = 0    
+        useST = 0
+        
+        for batchType in self.skychunk.batchTypes:
+            self.logger.infoPair("batchType", batchType)
+            if batchType == "ST":
+                useST = 1
+            if batchType == "P2":
+                useP2 = 1
+
+        if (useP2 ==0 and useST ==1):
+            #grab only stacks
+            cmd += " -photcode-start 11000 -photcode-end 11500"
+
+        if (useP2 ==1 and useST==0):
+            #grabd only P2s    
+            cmd += " -photcode-start 10000 -photcode-end 10577"
+
+
         self.logger.infoPair("Running dvopsps", cmd)
         p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)

branches/eam_branches/ipp-20131211/ippToPsps/jython/gpc1db.py

@@ -93 +93 @@
                    JOIN mergedvodbRun USING(minidvodb_id) \
                    JOIN mergedvodbProcessed USING (merge_id) \
-                   JOIN  stackRun USING(stack_id) JOIN skycell USING(skycell_id) \
+                   JOIN stackRun USING(stack_id) \
+                   JOIN stackSumSkyfile using (stack_id) \
+                   JOIN skycell USING(skycell_id) \
                    WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
                    AND minidvodbRun.state = 'merged' \
@@ -102 +104 @@
                    AND addRun.state = 'full' \
                    AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
-                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) 
-            
+                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+                   AND mjd_obs >= (to_days('" + tstart + "')-678941) \
+                   AND mjd_obs <= (to_days('" + tend + "') - 678941) "
+            self.logger.infoPair("sql",sql)      
         try:
             # XXX EAM : test output

branches/eam_branches/ipp-20131211/ippToPsps/jython/queue.py

@@ -45 +45 @@
 
         try:
+            print "why doesn't this work"
             self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
         except:

branches/eam_branches/ipp-20131211/ippToPsps/jython/stackbatch.py

@@ -303 +303 @@
             modelLong = model;
 
+        #does it have the EXT_COVAR in there?
+
+        sql = "DESCRIBE SkyChip_xfit 'EXT_COVAR_%'"
+        rs = self.scratchDb.executeQuery(sql)
+        rs.first()
+        hasResult = rs.next()
+        if hasResult:
+           print "EXT_COVARs exist"
+        else:
+           print "No EXT_COVARS here"
+
+        ext_covar_sql = ""
+  
+        if (hasResult):
+            ext_covar_sql = ", "+modelLong+"Covar11=b.EXT_COVAR_00_00,  \
+            "+modelLong+"Covar12=b.EXT_COVAR_00_01,  \
+            "+modelLong+"Covar13=b.EXT_COVAR_00_02,  \
+            "+modelLong+"Covar14=b.EXT_COVAR_00_03,  \
+            "+modelLong+"Covar15=b.EXT_COVAR_00_04,  \
+            "+modelLong+"Covar16=b.EXT_COVAR_00_05,  \
+            "+modelLong+"Covar17=b.EXT_COVAR_00_06,  \
+            "+modelLong+"Covar22=b.EXT_COVAR_01_01,  \
+            "+modelLong+"Covar23=b.EXT_COVAR_01_02,  \
+            "+modelLong+"Covar24=b.EXT_COVAR_01_03,  \
+            "+modelLong+"Covar25=b.EXT_COVAR_01_04,  \
+            "+modelLong+"Covar26=b.EXT_COVAR_01_05,  \
+            "+modelLong+"Covar27=b.EXT_COVAR_01_06,  \
+            "+modelLong+"Covar33=b.EXT_COVAR_02_02,  \
+            "+modelLong+"Covar34=b.EXT_COVAR_02_03,  \
+            "+modelLong+"Covar35=b.EXT_COVAR_02_04,  \
+            "+modelLong+"Covar36=b.EXT_COVAR_02_05,  \
+            "+modelLong+"Covar37=b.EXT_COVAR_02_06,  \
+            "+modelLong+"Covar44=b.EXT_COVAR_03_03,  \
+            "+modelLong+"Covar45=b.EXT_COVAR_03_04,  \
+            "+modelLong+"Covar46=b.EXT_COVAR_03_05,  \
+            "+modelLong+"Covar47=b.EXT_COVAR_03_06,  \
+            "+modelLong+"Covar55=b.EXT_COVAR_04_04,  \
+            "+modelLong+"Covar56=b.EXT_COVAR_04_05,  \
+            "+modelLong+"Covar57=b.EXT_COVAR_04_06,  \
+            "+modelLong+"Covar66=b.EXT_COVAR_05_05,  \
+            "+modelLong+"Covar67=b.EXT_COVAR_05_06,  \
+            "+modelLong+"Covar77=b.EXT_COVAR_06_06"
+       
+        
+
+
         sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \
         "+model+"Radius=b.EXT_WIDTH_MAJ,  \
@@ -308 +354 @@
         "+model+"FluxErr=ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " / 1.085736, \
         "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \
-        "+model+"Phi=b.EXT_THETA,  \
-        "+modelLong+"Covar11=b.EXT_COVAR_00_00,  \
-        "+modelLong+"Covar12=b.EXT_COVAR_00_01,  \
-        "+modelLong+"Covar13=b.EXT_COVAR_00_02,  \
-        "+modelLong+"Covar14=b.EXT_COVAR_00_03,  \
-        "+modelLong+"Covar15=b.EXT_COVAR_00_04,  \
-        "+modelLong+"Covar16=b.EXT_COVAR_00_05,  \
-        "+modelLong+"Covar17=b.EXT_COVAR_00_06,  \
-        "+modelLong+"Covar22=b.EXT_COVAR_01_01,  \
-        "+modelLong+"Covar23=b.EXT_COVAR_01_02,  \
-        "+modelLong+"Covar24=b.EXT_COVAR_01_03,  \
-        "+modelLong+"Covar25=b.EXT_COVAR_01_04,  \
-        "+modelLong+"Covar26=b.EXT_COVAR_01_05,  \
-        "+modelLong+"Covar27=b.EXT_COVAR_01_06,  \
-        "+modelLong+"Covar33=b.EXT_COVAR_02_02,  \
-        "+modelLong+"Covar34=b.EXT_COVAR_02_03,  \
-        "+modelLong+"Covar35=b.EXT_COVAR_02_04,  \
-        "+modelLong+"Covar36=b.EXT_COVAR_02_05,  \
-        "+modelLong+"Covar37=b.EXT_COVAR_02_06,  \
-        "+modelLong+"Covar44=b.EXT_COVAR_03_03,  \
-        "+modelLong+"Covar45=b.EXT_COVAR_03_04,  \
-        "+modelLong+"Covar46=b.EXT_COVAR_03_05,  \
-        "+modelLong+"Covar47=b.EXT_COVAR_03_06,  \
-        "+modelLong+"Covar55=b.EXT_COVAR_04_04,  \
-        "+modelLong+"Covar56=b.EXT_COVAR_04_05,  \
-        "+modelLong+"Covar57=b.EXT_COVAR_04_06,  \
-        "+modelLong+"Covar66=b.EXT_COVAR_05_05,  \
-        "+modelLong+"Covar67=b.EXT_COVAR_05_06,  \
-        "+modelLong+"Covar77=b.EXT_COVAR_06_06   \
+        "+model+"Phi=b.EXT_THETA  \
+        "+ext_covar_sql+"  \
         WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'"
 
+        print sql
         self.scratchDb.execute(sql)
 
         # sersic fit has an extra parameter
-        if ippModelType == "PS_MODEL_SERSIC":
+        if (ippModelType == "PS_MODEL_SERSIC" and hasResult > 0):
             sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \
             "+modelLong+"Covar18=b.EXT_COVAR_00_07,  \

branches/eam_branches/ipp-20131211/ippTools/share/Makefile.am

@@ -488 +488 @@
         releasetool_definerelgroup_select_exp_lap.sql \
         releasetool_stacksummary.sql \
-        releasetool_pendingrelgroup.sql
+        releasetool_pendingrelgroup.sql \
+        fftool_definebyquery.sql \
+        fftool_definebyquery_select_warps.sql \
+        fftool_result.sql \
+        fftool_revert.sql \
+        fftool_revertsummary.sql \
+        fftool_summary.sql \
+        fftool_todo.sql \
+        fftool_toadvance.sql
 

branches/eam_branches/ipp-20131211/ippTools/share/chiptool_pendingcleanuprun.sql

@@ -4 +4 @@
     rawExp.exp_tag,
     chipRun.state,
-    chipRun.workdir
+    chipRun.workdir,
+    chipRun.label,
+    IFNULL(Label.priority, 10000) AS priority
 FROM chipRun
 JOIN rawExp 
 USING (exp_id)
+LEFT JOIN Label ON chipRun.label = Label.label
 WHERE
     (chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_scrubbed' OR chipRun.state = 'goto_purged')

branches/eam_branches/ipp-20131211/ippTools/share/pxadmin_create_tables.sql

@@ -2307 +2307 @@
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 
+CREATE TABLE fullForceRun (
+    ff_id           BIGINT NOT NULL AUTO_INCREMENT,
+    skycal_id       BIGINT,
+    sources_path_base VARCHAR(255),
+    state           VARCHAR(64),
+    workdir         VARCHAR(255),
+    label           VARCHAR(64),
+    data_group      VARCHAR(64),
+    dist_group      VARCHAR(64),
+    note            VARCHAR(255),
+    reduction       VARCHAR(64),
+    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
+    PRIMARY KEY(ff_id),
+    KEY(state),
+    KEY(label),
+    KEY(data_group),
+    KEY(skycal_id),
+    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceInput (
+    ff_id           BIGINT,
+    warp_id         BIGINT,
+    PRIMARY KEY(ff_id, warp_id),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceResult (
+    ff_id           BIGINT,
+    warp_id         BIGINT,
+    path_base       VARCHAR(255) NOT NULL,
+    dtime_script    FLOAT,
+    quality         SMALLINT NOT NULL,
+    hostname        VARCHAR(64) NOT NULL,
+    software_ver    VARCHAR(16),
+    fault           SMALLINT NOT NULL,
+    PRIMARY KEY(ff_id, warp_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE fullForceSummary (
+    ff_id           BIGINT,
+    path_base       VARCHAR(255) NOT NULL,
+    dtime_script    FLOAT,
+    quality         SMALLINT NOT NULL,
+    hostname        VARCHAR(64) NOT NULL,
+    software_ver    VARCHAR(16),
+    fault           SMALLINT NOT NULL,
+    PRIMARY KEY(ff_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 
 -- These comment lines are here to avoid an empty query error.

branches/eam_branches/ipp-20131211/ippTools/share/pxadmin_drop_tables.sql

@@ -120 +120 @@
 DROP TABLE IF EXISTS dqstatsContent;
 DROP TABLE IF EXISTS dqstatsRun;
+DROP TABLE IF EXISTS fullForceRun;
+DROP TABLE IF EXISTS fullForceInput;
+DROP TABLE IF EXISTS fullForceResult;
+DROP TABLE IF EXISTS fullForceSummary;
+
 
 SET FOREIGN_KEY_CHECKS=1

branches/eam_branches/ipp-20131211/ippTools/src

@@ -47 +47 @@
 sctool
 releasetool
+fftool

@@ -47 +47 @@
 sctool
 releasetool
+fftool

branches/eam_branches/ipp-20131211/ippTools/src/Makefile.am

@@ -33 +33 @@
         vptool \
         sctool \
-        releasetool
+        releasetool \
+        fftool
 
 pkginclude_HEADERS = \
@@ -87 +88 @@
         vptool.h \
         sctool.h \
-        releasetool.h
+        releasetool.h \
+        fftool.h
 
 lib_LTLIBRARIES = libpxtools.la
@@ -332 +334 @@
     releasetoolConfig.c
 
+fftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+fftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+fftool_SOURCES = \
+    fftool.c \
+    fftoolConfig.c
+
 clean-local:
         -rm -f TAGS

branches/eam_branches/ipp-20131211/ippTools/src/chiptool.c

@@ -1255 +1255 @@
 
     psMetadata *where = psMetadataAlloc();
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "==");
 
     psString query = pxDataGet("chiptool_pendingcleanuprun.sql");
@@ -1269 +1269 @@
     }
     psFree(where);
+
+    psStringAppend(&query, "\nORDER BY priority DESC, chip_id");
 
     // treat limit == 0 as "no limit"

branches/eam_branches/ipp-20131211/ippconfig/gpc1/psastro.config

@@ -380 +380 @@
 END
 
+POLE_REFCAT METADATA
+   PSASTRO.CATDIR               STR  SYNTH.GRIZY
+   ZERO.POINT.USE.MEAN               BOOL TRUE  
+   PSASTRO.GRID.MIN.ANGLE       F32  -20
+   PSASTRO.GRID.MAX.ANGLE       F32  +20
+   PSASTRO.GRID.DEL.ANGLE       F32   0.5
+   PSASTRO.FIELD.PADDING        F32   0.5
+END
+
 TEST_REFCAT METADATA
    PSASTRO.CATDIR               STR /data/ipp064.0/ipp/ippRefs/catdir.refcat.20120524.v0

branches/eam_branches/ipp-20131211/ippconfig/recipes/filerules-split.mdc

@@ -262 +262 @@
 PSPHOT.STACK.RESID           OUTPUT {OUTPUT}.stk.{FILE.ID}.res.fits   IMAGE           COMP_SUB   FPA        TRUE      NONE
 PSPHOT.STACK.CONFIG          OUTPUT {OUTPUT}.psphotStack.mdc          TEXT            NONE       FPA        TRUE      NONE
+
+PSPHOT.FULLFORCE.OUTPUT      OUTPUT {OUTPUT}.cmf                      CMF             NONE       FPA        TRUE      NONE
                                                      
 SOURCE.PLOT.RAW.MOMENTS      OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png      KAPA            NONE       CHIP       TRUE      NONE

branches/eam_branches/ipp-20131211/ippconfig/recipes/psastro.config

@@ -3 +3 @@
 PSASTRO.ONLY.REFSTARS      BOOL FALSE  # skip all but refstar matches
 PSASTRO.SAVE.REFMATCH      BOOL FALSE  # save refstar matches as table in output smf file
+PSASTRO.SAVE.CFF           BOOL FALSE  # save a cff file (input for psphotFullForce)
 
 # select which WCS style to use on output images.
@@ -272 +273 @@
 END
 
+POLE_REFCAT METADATA
+END
+
 TEST_REFCAT METADATA
 END
 
 STATICSKY_CAL   METADATA
+    PSASTRO.SAVE.CFF           BOOL TRUE  # save a cff file (input for psphotFullForce)
 END
 

branches/eam_branches/ipp-20131211/ippconfig/recipes/psphot.config

@@ -196 +196 @@
 EXT.NSIGMA.LIMIT.USE                BOOL  TRUE
 
-EXT.FIT.MIN.GAL.LIMIT               F32   10.0
-EXT.FIT.MIN.GAL.LIMIT.USE           BOOL  FALSE
+# Limits on extended source fits by galactic coordinates 
+EXT.FIT.MIN.GAL.LIMIT.USE           BOOL  FALSE # if true apply the galactic latitude cut
+# objects are skipped for extended source fits if they have a abs(galactic latitude) larger than
+# a limit value which is a function of galactic longitude.
+#     The galactic latitude limit is the sum of the following constant value ...
+EXT.FIT.MIN.GAL.LIMIT               F32   20.0
+# ... and a gaussian function of galactic longitude centered at b = 0 with this magnitude
+EXT.FIT.MIN.GAL.LIMIT.BULGE         F32   15.0  
+# ... and this sigma
+EXT.FIT.MIN.GAL.LIMIT.BULGE.SIGMA   F32   50.0  # with this sigma value
+
 
 KRON_ITERATIONS                     S32   2
@@ -224 +233 @@
  #QGAUSS_PSF  EXTENDED_SOURCE_MODEL  PS_MODEL_QGAUSS   20.0    TRUE
  # TRAIL_RAW EXTENDED_SOURCE_MODEL  PS_MODEL_TRAIL    10.0   FALSE
- EXP_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_EXP      10.0    TRUE
- DEV_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_DEV      10.0    TRUE
- SER_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_SERSIC   10.0    TRUE
+ EXP_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_EXP      0.0    TRUE
+ DEV_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_DEV      0.0    TRUE
+ SER_PCM  EXTENDED_SOURCE_MODEL  PS_MODEL_SERSIC   0.0    TRUE
 END
 
@@ -407 +416 @@
 
 EXT.ANALYSIS.MAG.LIMITS METADATA
-    TYPE  DATA FILTER.ID MAG.LIMIT
-    gband DATA g         NAN            
-    rband DATA r         NAN             
-    iband DATA i         NAN             
-    zband DATA z         NAN             
-    yband DATA y         NAN             
-    wband DATA w         NAN             
-    other DATA any       NAN             
+    TYPE  DATA FILTER.ID MAG.LIMIT.PETRO    MAG.LIMIT.EXTFIT
+    gband DATA g         25                 21.5
+    rband DATA r         25                 21.5 
+    iband DATA i         25                 21.5
+    zband DATA z         25                 20.5
+    yband DATA y         25                 19.5
+    wband DATA w         25                 21.5
+    other DATA any       25                 21.5
 END
 
@@ -423 +432 @@
 RADIAL_APERTURES_SN_LIM             F32   0.0  # S/N limit for radial aperture calculation
 
+# zero point and exposure time to which fullForceSummary scales fluxes to
+PSPHOT.FULLFORCE.EXPTIME            F32   1.00 
+PSPHOT.FULLFORCE.ZERO_PT            F32   25.00
+
+
 GALAXY_SHAPES                       BOOL  F
+GALAXY_SHAPES_STYLE                 STR   R_MAJ_R_MIN
+
 GALAXY_SHAPES_FR_MAJOR_MIN          F32   0.5
-GALAXY_SHAPES_FR_MAJOR_MAX          F32   2.0
-GALAXY_SHAPES_FR_MAJOR_DEL          F32   0.1
+GALAXY_SHAPES_FR_MAJOR_MAX          F32   2.00
+GALAXY_SHAPES_FR_MAJOR_DEL          F32   0.05
 GALAXY_SHAPES_FR_MINOR_MIN          F32   0.5
-GALAXY_SHAPES_FR_MINOR_MAX          F32   2.0
-GALAXY_SHAPES_FR_MINOR_DEL          F32   0.1
-
+GALAXY_SHAPES_FR_MINOR_MAX          F32   2.00
+GALAXY_SHAPES_FR_MINOR_DEL          F32   0.05
+
+#GALAXY_SHAPES_FR_MAJOR_MIN          F32   0.85
+#GALAXY_SHAPES_FR_MAJOR_MAX          F32   1.15
+#GALAXY_SHAPES_FR_MAJOR_DEL          F32   0.05
+#GALAXY_SHAPES_FR_MINOR_MIN          F32   0.85
+#GALAXY_SHAPES_FR_MINOR_MAX          F32   1.15
+#GALAXY_SHAPES_FR_MINOR_DEL          F32   0.05
 
 # Extended source fit parameters
@@ -439 +461 @@
   EXTENDED_SOURCE_ANNULI              BOOL  TRUE
   EXT.NSIGMA.LIMIT.USE                BOOL  FALSE
+  EXT.FIT.MIN.GAL.LIMIT.USE           BOOL  TRUE  # limit extended source fits by galactic coordinates
 
   PSPHOT.STACK.MATCH.PSF.SOURCE       STR   AUTO # which inputs to convolve? (RAW, CNV, AUTO)
@@ -465 +488 @@
   EXT_MODEL                           STR   PS_MODEL_QGAUSS
   PEAKS_NMAX_TOTAL                    S32   0 # set this to limit the allowed number of peaks - Yields fault instead of avoid memory explosion
+
 
   SAVE.RESID                          BOOL  TRUE
@@ -683 +707 @@
     RADIAL_APERTURES        BOOL    TRUE
     EXTENDED_SOURCE_FITS    BOOL    TRUE    # this casues the xfit extension to be written out
-    EXTENDED_SOURCE_PETROSIAN BOOL    TRUE  # I want petrosian mags
+    EXTENDED_SOURCE_PETROSIAN BOOL  TRUE    # To measure petrosian magnitudes
+    EXTENDED_SOURCE_ANNULI  BOOL    TRUE
     SAVE.PSF                BOOL    FALSE
     SAVE.BACKMDL            BOOL    FALSE
-    # SAVE.RESID            BOOL    FALSE
-    # OUTPUT.FORMAT         STR     PS1_V3
-END
+    SAVE.RESID              BOOL    TRUE
+    OUTPUT.FORMAT           STR     PS1_SV2
+END

branches/eam_branches/ipp-20131211/ippconfig/recipes/reductionClasses.mdc

@@ -251 +251 @@
         BACKGROUND_PSWARP       STR     BACKGROUND
         FULLFORCE_PSPHOT  STR   FULLFORCE_WARP
+END
+
+# basic science analysis
+POLE_REFCAT             METADATA
+        CHIP_PPIMAGE      STR     CHIP
+        CHIP_PSPHOT       STR     CHIP
+        JPEG_BIN1         STR     PPIMAGE_J1
+        JPEG_BIN2         STR     PPIMAGE_J2
+        ADDSTAR           STR     ADDSTAR
+        PSASTRO           STR     POLE_REFCAT
 END
 

branches/eam_branches/ipp-20131211/ppSub/src/ppSubMatchPSFs.c

@@ -156 +156 @@
     psLogMsg("ppSub", PS_LOG_INFO, "Input FWHM: %f\nReference FWHM: %f\n", inFWHM, refFWHM);
     if (!isfinite(inFWHM) || !isfinite(refFWHM)) {
-        psError(PPSUB_ERR_DATA, false, "Cannot determine FHWM for images, giving up.");
-        return false;
+        psErrorStackPrint(stderr, "Cannot determine FHWM for images, giving up.");
+        int error = psErrorCodeLast(); // Error code
+        ppSubDataQuality(data, error, PPSUB_FILES_ALL);
+        return true;
     }
 

branches/eam_branches/ipp-20131211/psLib/src/math/psMixtureModels.c

@@ -120 +120 @@
   *Ncensored = 0;
 
+  int k = 0;
   for (int i = 0; i < in->numRows; i++) {
     int isCensored = 0;
@@ -130 +131 @@
       *Ncensored = *Ncensored + 1;
     }
-    offsets->data.S32[i] = *Ncensored;
-  }
+    else {
+      offsets->data.S32[k] = i - k;
+      k++;
+    }
+    //    offsets->data.S32[i] = isCensored;
+  }
+
   psImage *out;
   if (*Ncensored == 0) {
@@ -320 +326 @@
         }
       }
-      i = modes->data.F32[k];
-      counts->data.F32[i] += 1.0;
+      if (modes->data.F32[k] != -1) {
+        i = modes->data.F32[k];
+        counts->data.F32[i] += 1.0;
+      }
     }
 
@@ -335 +343 @@
     for (k = 0; k < N; k++) {
       i = modes->data.F32[k];
-      for (j = 0; j < dim; j++) {
-        means->data.F32[i][j] += D->data.F32[k][j];
+      if (i != -1) {
+        for (j = 0; j < dim; j++) {
+          means->data.F32[i][j] += D->data.F32[k][j];
+        }
       }
     }

branches/eam_branches/ipp-20131211/psModules/src/camera/pmFPAfile.c

@@ -565 +565 @@
       case PM_FPA_FILE_CMF:
         return ("CMF");
+      case PM_FPA_FILE_CFF:
+        return ("CFF");
       case PM_FPA_FILE_WCS:
         return ("WCS");

branches/eam_branches/ipp-20131211/psModules/src/objects/pmModelClass.c

@@ -66 +66 @@
 
 static pmModelClass *models = NULL;
+static psVector *modelClassLookupTable = NULL;  // translation between model types in header and here
 static int Nmodels = 0;
 
@@ -135 +136 @@
     models = NULL;
     Nmodels = 0;
+    psFree(modelClassLookupTable);
+    modelClassLookupTable = NULL;
     return;
 }
@@ -193 +196 @@
 }
 
+
+bool pmModelClassWriteHeader(psMetadata *header)
+{
+    psMetadataAddS32(header, PS_LIST_TAIL, "MTNUM", PS_META_REPLACE, "number of model types", Nmodels);
+    for (int i = 0; i < Nmodels; i++) {
+        char modelNameKey[16];
+        char modelValKey[16];
+        sprintf(modelNameKey, "MTNAM%02d", i);
+        sprintf(modelValKey,  "MTVAL%02d", i);
+        psMetadataAddStr(header, PS_LIST_TAIL, modelNameKey, PS_META_REPLACE, "", models[i].name);
+        psMetadataAddS32(header, PS_LIST_TAIL, modelValKey, PS_META_REPLACE, "", i);
+    }
+
+    return true;
+}
+
+bool pmModelClassReadHeader(psMetadata *header) {
+    psFree(modelClassLookupTable);
+
+    bool status;
+    int numHeaderModels = psMetadataLookupS32(&status, header, "MTNUM");
+    if (!status) {
+        return false;
+    }
+
+    psVector *inputTypes = psVectorAlloc(numHeaderModels, PS_TYPE_S32);
+    psVector *localTypes = psVectorAlloc(numHeaderModels, PS_TYPE_S32);
+    int max_val = -1;
+    for (int i = 0; i < numHeaderModels; i++) {
+        char modelNameKey[16];
+        char modelValKey[16];
+        sprintf(modelNameKey, "MTNAM%02d", i);
+        sprintf(modelValKey,  "MTVAL%02d", i);
+        psString thisName = psMetadataLookupStr(&status, header, modelNameKey);
+        int thisVal = psMetadataLookupS32(&status, header, modelValKey);
+        if (thisVal > max_val) {
+            max_val = thisVal;
+        }
+        inputTypes->data.S32[i] = thisVal;
+        localTypes->data.S32[i] = pmModelClassGetType(thisName);
+    }
+    if (max_val < 0) {
+        psFree(inputTypes);
+        psFree(localTypes);
+        return false;
+    }
+
+    modelClassLookupTable = psVectorAlloc(max_val + 1, PS_TYPE_S32);
+    psVectorInit(modelClassLookupTable, -1);
+
+    for (int i = 0; i < numHeaderModels; i++) {
+        int thisVal = inputTypes->data.S32[i];
+        int localVal = localTypes->data.S32[i];
+        modelClassLookupTable->data.S32[thisVal] = localVal;
+    }
+    psFree(inputTypes);
+    psFree(localTypes);
+
+    return true;
+}
+
+pmModelType pmModelClassGetLocalType(pmModelType inputType) {
+    pmModelType localType = -1;
+
+    if (modelClassLookupTable) {
+        if (inputType >= 0 && inputType < modelClassLookupTable->n) {
+            localType = modelClassLookupTable->data.S32[inputType];
+        }
+    } else {
+        // no lookup table defined
+        // for backwards compatability if inputType refers to a defined model, return it
+        if (inputType >= 0 && pmModelClassGetName(inputType)) {
+            localType = inputType;
+        }
+    }
+
+    return localType;
+}

branches/eam_branches/ipp-20131211/psModules/src/objects/pmModelClass.h

@@ -76 +76 @@
 void pmModelClassSetLimits(pmModelLimitsType type);
 
+// write keywords to header definining the model type values used by this program
+bool pmModelClassWriteHeader(psMetadata *header);
+// create a lookup table for translating input model type values to local model type values
+bool pmModelClassReadHeader(psMetadata *header);
+// translate input model type value to local value
+pmModelType pmModelClassGetLocalType(pmModelType inputType);
 
 /// @}

branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO.c

@@ -61 +61 @@
 static bool pmReadoutReadXFIT(pmFPAfile *file, pmReadout *readout, char * exttype, psMetadata *hduHeader, psString xfitname, psArray *sources, long *sourceIndex);
 static bool pmReadoutReadXRAD(pmFPAfile *file, pmReadout *readout, char * exttype, psMetadata *hduHeader, psString xfitname, psArray *sources, long *sourceIndex);
+static bool pmReadoutReadXGAL(pmFPAfile *file, pmReadout *readout, char * exttype, psMetadata *hduHeader, psString xfitname, psArray *sources, long *sourceIndex);
 
 // lookup the EXTNAME values used for table data and image header segments
@@ -374 +375 @@
         }                                                               \
         if (xgalname) {                                                 \
-            status &= pmSourcesWrite_##TYPE##_XGAL (file->fits, sources, xgalname, recipe); \
+            status &= pmSourcesWrite_##TYPE##_XGAL (file->fits, readout, sources, xgalname, recipe); \
         }                                                               \
     }
@@ -1036 +1037 @@
         bool XFIT_OUTPUT = psMetadataLookupBool(&status, recipe, "EXTENDED_SOURCE_FITS");
         bool XRAD_OUTPUT = psMetadataLookupBool(&status, recipe, "RADIAL_APERTURES");
-        bool XGAL_OUTPUT = false; // psMetadataLookupBool(&status, recipe, "GALAXY_SHAPES");
+        bool XGAL_OUTPUT = psMetadataLookupBool(&status, recipe, "GALAXY_SHAPES");
 
         if (!pmSourceIOextnames(&headname, &dataname, &deteffname, 
@@ -1122 +1123 @@
 
             long *sourceIndex = NULL;
-            if (XSRC_OUTPUT || XFIT_OUTPUT || XRAD_OUTPUT) {
+            if (XSRC_OUTPUT || XFIT_OUTPUT || XRAD_OUTPUT || XGAL_OUTPUT) {
+                // Build sourceIndex. Lookup table from source->seq to index in sources array.
+                // Consists of an array of length max(source->seq) + 1.
+
+                // find maximum sequence number
                 long seq_max = -1;
                 for (long i = sources->n -1; i >= 0; i--) {
@@ -1135 +1140 @@
                     }
                 }
+                // allocate and initialize the index
                 sourceIndex = psAlloc((seq_max + 1) * sizeof(long));
                 for (long i = 0; i < seq_max; i++) {
                     sourceIndex[i] = -1;
                 }
+                // populate the index
                 for (long i = 0; i < sources->n; i++) {
                     pmSource *source = sources->data[i];
@@ -1165 +1172 @@
                 psFree(xradname);
             }
+            if (XGAL_OUTPUT && xgalname) {
+                // a cmf file may have an XGAL extension, but it is not required
+                if (!pmReadoutReadXGAL(file, readout, exttype, hdu->header, xgalname, sources, sourceIndex)) {
+                    // do anything?
+                }
+                psFree(xgalname);
+            }
             psFree(sourceIndex);
 
@@ -1461 +1475 @@
     return status;
 }
+static bool pmReadoutReadXGAL(pmFPAfile *file, pmReadout *readout, char *exttype, psMetadata *hduHeader, psString xgalname, psArray *sources, long *sourceIndex) 
+{
+    if (!psFitsMoveExtNameClean (file->fits, xgalname)) {
+        psTrace ("pmFPAfile", 1, "cannot find xgal extension %s in %s, skipping", xgalname, file->filename);
+        return false;
+    }
+
+    psMetadata *tableHeader = psFitsReadHeader(NULL, file->fits); // The FITS header
+    if (!tableHeader) psAbort("cannot read table header");
+
+    char *xtension = psMetadataLookupStr (NULL, tableHeader, "XTENSION");
+    if (!xtension) psAbort("cannot read table type");
+    if (strcmp (xtension, "BINTABLE")) {
+        psFree(tableHeader);
+        psWarning ("no binary table in extension %s, skipping\n", xgalname);
+        return false;
+    }
+
+# define PM_SOURCES_READ_XGAL(NAME,TYPE)                                \
+    if (!strcmp (exttype, NAME)) {                                      \
+        status = pmSourcesRead_##TYPE##_XGAL(file->fits, readout, hduHeader, tableHeader, sources, sourceIndex); \
+    }                                                                   
+
+    bool status = false;
+    if (file->type == PM_FPA_FILE_CMF) {
+        PM_SOURCES_READ_XGAL("PS1_V1",    CMF_PS1_V1);
+        PM_SOURCES_READ_XGAL("PS1_V2",    CMF_PS1_V2);
+        PM_SOURCES_READ_XGAL("PS1_V3",    CMF_PS1_V3);
+        PM_SOURCES_READ_XGAL("PS1_V4",    CMF_PS1_V4);
+        PM_SOURCES_READ_XGAL("PS1_SV1",   CMF_PS1_SV1);
+        PM_SOURCES_READ_XGAL("PS1_SV2",   CMF_PS1_SV2);
+        PM_SOURCES_READ_XGAL("PS1_DV1",   CMF_PS1_DV1);
+        PM_SOURCES_READ_XGAL("PS1_DV2",   CMF_PS1_DV2);
+        PM_SOURCES_READ_XGAL("PS1_DV3",   CMF_PS1_DV3);
+    }
+    psFree(tableHeader);
+    return status;
+}

branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO.h

@@ -21 +21 @@
   bool pmSourcesWrite_##TYPE##_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); \
   bool pmSourcesWrite_##TYPE##_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); \
-  bool pmSourcesWrite_##TYPE##_XGAL(psFits *fits, psArray *sources, char *extname, psMetadata *recipe); \
+  bool pmSourcesWrite_##TYPE##_XGAL(psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe); \
   psArray *pmSourcesRead_##TYPE (psFits *fits, psMetadata *header); \
   bool pmSourcesRead_##TYPE##_XSRC (psFits *fits, pmReadout *readout, psMetadata *header, psMetadata *tableHeader, psArray *sources, long *index); \
   bool pmSourcesRead_##TYPE##_XFIT (psFits *fits, pmReadout *readout, psMetadata *header, psMetadata *tableHeader, psArray *sources, long *index); \
   bool pmSourcesRead_##TYPE##_XRAD (psFits *fits, pmReadout *readout, psMetadata *header, psMetadata *tableHeader, psArray *sources, long *index);\
+  bool pmSourcesRead_##TYPE##_XGAL (psFits *fits, pmReadout *readout, psMetadata *header, psMetadata *tableHeader, psArray *sources, long *index);\
   
 // All of these functions need to use the same API, even if not all elements are used in a specific case

branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_CFF.c

@@ -65 +65 @@
     int modelType = pmModelClassGetType ("PS_MODEL_GAUSS");
 
+    // Read lookup table for model classes (if defined)
+    pmModelClassReadHeader(header);
+
     char *PSF_NAME = psMetadataLookupStr (&status, header, "PSFMODEL");
     if (PSF_NAME != NULL) {
@@ -111 +114 @@
         float theta      = psMetadataLookupF32 (&status, row, "THETA");
 
-        // XXX: we need to put a lookup table in the cff header to define the correspondence of the
-        // model type values in the cff with our models. (We want to use an interger for efficiency
-        // but the value for each model type is set on the organization of the the array in pmModelClass.c
-        // For now use the input values verbatim and trust the user that this is valid value
         int   galaxyModelType = psMetadataLookupS32(&status, row, "MODEL_TYPE");
+        if (status) {
+            galaxyModelType = pmModelClassGetLocalType(galaxyModelType);
+        } else {
+            galaxyModelType = -1;
+        }
         float Sindex     = psMetadataLookupF32 (&status, row, "INDEX"); // Should this be PAR_07 not sersic index
 
@@ -123 +127 @@
         source->type = PM_SOURCE_TYPE_STAR; // XXX this should be added to the flags
 
-        // XXX we can set this in general, but for a specific image, we need to weed out SATSTARS
+        // XXX we can set this in general, but for a specific image, we need to weed out SATSTARS and
+        // stars that are masked
         if (psfStar) {
             source->tmpFlags |= PM_SOURCE_TMPF_CANDIDATE_PSFSTAR;
@@ -180 +185 @@
         }
 
-        // XXX: should use < 0 as invalid galaxyModelType
-
-        if (fitGalaxy && galaxyModelType > 0) {
+        if (fitGalaxy && galaxyModelType >= 0) {
             source->modelFits = psArrayAllocEmpty (1);
             pmModel *model = pmModelAlloc(galaxyModelType);
@@ -235 +238 @@
     PS_ASSERT(mdok, false);
 
+    // write the definition of the model class type values to the header
+    psMetadata *outputHeader = psMetadataAlloc();
+    pmModelClassWriteHeader(outputHeader);
+
     psArray *table = psArrayAllocEmpty(sources->n);
 
@@ -246 +253 @@
         psS32 modelType = 0;
         bool fitGalaxy = false;
-        bool psfStar = false;
+        bool psfStar = (source->mode & PM_SOURCE_MODE_PSFSTAR) ? true : false;
         psF32 sersicIndex = 0;
-        if (source->modelFits == NULL) {
+        // For now only perform galaxy fits on extended objects
+        if (source->modelEXT == NULL) {
             pmModel *model = source->modelPSF;
             if (model == NULL) continue;
@@ -260 +268 @@
             yPos = model->params->data.F32[PM_PAR_YPOS];
             flux = source->psfFlux;
-            psfStar = (source->mode & PM_SOURCE_MODE_PSFSTAR) ? true : false;
             rMajor = 0;
             rMinor = 0;
@@ -329 +336 @@
 
         psArrayAdd(table, 100, row);
+        psFree(row);
     }
 
-    if (!psFitsWriteTable(fits, NULL, table, extname)) {
+    if (!psFitsWriteTable(fits, outputHeader, table, extname)) {
         psError(psErrorCodeLast(), false, "writing ext data %s\n", extname);
         psFree(table);
-        psFree(header);
+        psFree(outputHeader);
         return false;
     }
     psFree(table);
-    // psFree(header);
+    psFree(outputHeader);
 
     return true;

branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_CMF.c.in

@@ -708 +708 @@
             return false;
         }
-        // Find the source with this sequence number. 
-        // XXX: I am assuming that sources is sorted in order of seq
+        // Find the source with this sequence number using the sourceIndex. 
         long seq = psMetadataLookupU32 (&status, row, "IPP_IDET");
-        pmSource *source = NULL;
-#ifndef ASSUME_SORTED
-        long j = seq < sources->n ? seq : sources->n - 1;
-        for (; j >= 0; j--) {
-            source = sources->data[j];
-            if (source->seq == seq) {
-                break;
-            }
-        }
-#else
         long j = sourceIndex[seq];
         psAssert(j >= 0 && j < sources->n, "invalid sourceIndex");
-        source = sources->data[j];
-#endif
+        pmSource *source = sources->data[j];
         if (!source) {
             psError(PS_ERR_UNKNOWN, false, "Failed to find source for row %ld sequence number %ld\n", i, seq);
@@ -793 +781 @@
     // create a header to hold the output data
     psMetadata *outhead = psMetadataAlloc ();
+
+    pmModelClassWriteHeader(outhead);
 
     // write the links to the image header
@@ -1034 +1024 @@
         return false;
     }
+    // set up the lookup table to translate between input model types and output model types
+    // if not defined it is assumed that the tables are the same
+    pmModelClassReadHeader(tableHeader);
 
     for (long i = 0; i < numSources; i++) {
@@ -1042 +1035 @@
             return false;
         }
-        // Find the source with this sequence number. 
-        // XXX: I am assuming that sources is sorted in order of seq.
         long seq = psMetadataLookupU32 (&status, row, "IPP_IDET");
-        long j = seq < sources->n ? seq : sources->n - 1;
-        pmSource *source = NULL;
-        for (; j >= 0; j--) {
-            source = sources->data[j];
-            if (source->seq == seq) {
-                break;
-            }
-        }
+        long j = sourceIndex[seq];
+        psAssert(j >= 0 && j < sources->n, "invalid sourceIndex");
+        pmSource *source = sources->data[j];
         if (!source) {
             psError(PS_ERR_UNKNOWN, false, "Failed to find source for row %ld sequence number %ld\n", i, seq);
@@ -1095 +1081 @@
         // in the psf table.
         psS32 extModelType = psMetadataLookupS32(&status, row, "EXT_MODEL_TYPE");
-        if (!status) {
+        if (status) {
+            // translate between the type value in xfit and values used by this program
+            extModelType = pmModelClassGetLocalType(extModelType);
+        } else {
             // older cmfs don't have this column
             extModelType = -1;
-        }
+        } 
 
         psEllipseAxes axes;
@@ -1339 +1328 @@
             return false;
         }
-        // Find the source with this sequence number. 
-        // XXX: I am assuming that sources is sorted in order of seq.
         long seq = psMetadataLookupU32 (&status, row, "IPP_IDET");
-        long j = seq < sources->n ? seq : sources->n - 1;
-        pmSource *source = NULL;
-        for (; j >= 0; j--) {
-            source = sources->data[j];
-            if (source->seq == seq) {
-                break;
-            }
-        }
+        long j = sourceIndex[seq];
+        psAssert(j >= 0 && j < sources->n, "invalid sourceIndex");
+        pmSource *source = sources->data[j];
         if (!source) {
             psError(PS_ERR_UNKNOWN, false, "Failed to find source for row %ld sequence number %ld\n", i, seq);
@@ -1407 +1389 @@
 
 // XXX where should I record the number of columns??
-bool pmSourcesWrite_CMF_@CMFMODE@_XGAL (psFits *fits, psArray *sources, char *extname, psMetadata *recipe)
+bool pmSourcesWrite_CMF_@CMFMODE@_XGAL (psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe)
 {
     bool status = false;
@@ -1422 +1404 @@
     // write the links to the image header
     psMetadataAddStr (outhead, PS_LIST_TAIL, "EXTNAME", PS_META_REPLACE, "galaxy table extension", extname);
+
+    psMetadataAddStr (outhead, PS_LIST_TAIL, "HI", PS_META_REPLACE, "does this get through?", "THERE");
 
     // let's write these out in S/N order
@@ -1491 +1475 @@
 }
 
+bool pmSourcesRead_CMF_@CMFMODE@_XGAL(psFits *fits, pmReadout *readout, psMetadata *hduHeader, psMetadata *tableHeader, psArray *sources, long *sourceIndex)
+{
+    PS_ASSERT_PTR_NON_NULL(fits, false);
+    PS_ASSERT_PTR_NON_NULL(sources, false);
+
+    bool status;
+    long numSources = psFitsTableSize(fits); // Number of sources in table
+    if (numSources == 0) {
+        psError(psErrorCodeLast(), false, "XGAL Table contains no entries\n");
+        return false;
+    }
+
+    for (long i = 0; i < numSources; i++) {
+        psMetadata *row = psFitsReadTableRow(fits, i); // Table row
+        if (!row) {
+            psError(psErrorCodeLast(), false, "Unable to read row %ld of sources", i);
+            psFree(row);
+            return false;
+        }
+        // Find the source with this sequence number. 
+        // XXX: I am assuming that sources is sorted in order of seq
+        long seq = psMetadataLookupU32 (&status, row, "IPP_IDET");
+        long j = sourceIndex[seq];
+        psAssert(j >= 0 && j < sources->n, "invalid sourceIndex");
+
+        pmSource *source = sources->data[j];
+        if (!source) {
+            psError(PS_ERR_UNKNOWN, false, "Failed to find source for row %ld sequence number %ld\n", i, seq);
+            psFree(row);
+            return false;
+        }
+
+        psVector *Flux  = psMetadataLookupVector(&status, row, "GAL_FLUX");
+        psVector *dFlux = psMetadataLookupVector(&status, row, "GAL_FLUX_ERR");
+        psVector *chisq = psMetadataLookupVector(&status, row, "GAL_CHISQ");
+
+        if (Flux && Flux->n > 0) {
+            psFree(source->galaxyFits);
+            source->galaxyFits = pmSourceGalaxyFitsAlloc();
+            source->galaxyFits->nPix = psMetadataLookupF32(&status, row, "NPIX");
+
+            psFree(source->galaxyFits->Flux);
+            source->galaxyFits->Flux  = psMemIncrRefCounter(Flux);
+            psFree(source->galaxyFits->dFlux);
+            source->galaxyFits->dFlux = psMemIncrRefCounter(dFlux);
+            psFree(source->galaxyFits->chisq);
+            source->galaxyFits->chisq = psMemIncrRefCounter(chisq);
+        }
+
+        psFree(row);
+    }
+
+    return true;
+}

branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_CAL_0.c

@@ -714 +714 @@
 }
 
-bool pmSourcesWrite_PS1_CAL_0_XGAL (psFits *fits, psArray *sources, char *extname, psMetadata *recipe)
+bool pmSourcesWrite_PS1_CAL_0_XGAL (psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe)
 {
     return true;

branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_DEV_0.c

@@ -256 +256 @@
 }
 
-bool pmSourcesWrite_PS1_DEV_0_XGAL(psFits *fits, psArray *sources, char *extname, psMetadata *recipe)
-{
-    return true;
-}
+bool pmSourcesWrite_PS1_DEV_0_XGAL(psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe)
+{
+    return true;
+}

branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_DEV_1.c

@@ -596 +596 @@
 }
 
-bool pmSourcesWrite_PS1_DEV_1_XGAL(psFits *fits, psArray *sources, char *extname, psMetadata *recipe)
+bool pmSourcesWrite_PS1_DEV_1_XGAL(psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe)
 {
     return true;

branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_SMPDATA.c

@@ -226 +226 @@
 } 
 
-bool pmSourcesWrite_SMPDATA_XGAL(psFits *fits, psArray *sources, char *extname, psMetadata *recipe)
+bool pmSourcesWrite_SMPDATA_XGAL(psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe)
 {
     return true;

branches/eam_branches/ipp-20131211/psastro/src/psastroDataSave.c

@@ -42 +42 @@
     }
 
-    // de-activate all files except PSASTRO.OUTPUT
+    // de-activate all files except PSASTRO.OUTPUT, PSASTRO.OUT.ASTROM, and PSPHOT.OUTPUT.CFF
     pmFPAfileActivate (config->files, false, NULL);
     pmFPAfileActivate (config->files, true, "PSASTRO.OUTPUT");
     pmFPAfileActivate (config->files, true, "PSASTRO.OUT.ASTROM");
+    pmFPAfileActivate (config->files, true, "PSPHOT.OUTPUT.CFF");
 
     pmFPAview *view = pmFPAviewAlloc (0);
@@ -103 +104 @@
     }
 
-    // activate all files except PSASTRO.OUTPUT
+    // activate all files except PSASTRO.OUTPUT, and PSPHOT.OUTPUT.CFF
     pmFPAfileActivate (config->files, true, NULL);
     pmFPAfileActivate (config->files, false, "PSASTRO.OUTPUT");
+    pmFPAfileActivate (config->files, false, "PSPHOT.OUTPUT.CFF");
 
     psFree (view);

branches/eam_branches/ipp-20131211/psastro/src/psastroDefineFiles.c

@@ -100 +100 @@
     }
 
+    bool writeCff = psMetadataLookupBool (&status, recipe, "PSASTRO.SAVE.CFF");
+    if (writeCff) {
+        pmFPAfile *file = pmFPAfileDefineOutputFromFile  (config, input, "PSPHOT.OUTPUT.CFF");
+        if (!file) {
+            psError (PS_ERR_IO, false, "Can't find the output cff file definition");
+            return NULL;
+        }
+        if (file->type != PM_FPA_FILE_CFF) {
+            psError(PS_ERR_IO, true, "%s is not of type %s", "PSPHOT.OUTPUT.CFF", pmFPAfileStringFromType (PM_FPA_FILE_CFF));
+            return NULL;
+        }
+        file->save = true;
+    }
+
 
 # if (0)

branches/eam_branches/ipp-20131211/psphot/src

@@ -26 +26 @@
 psphotFullForce
 psmakecff
+psphotFullForceSummary

@@ -26 +26 @@
 psphotFullForce
 psmakecff
+psphotFullForceSummary

branches/eam_branches/ipp-20131211/psphot/src/Makefile.am

@@ -25 +25 @@
 libpsphot_la_LDFLAGS = $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 
-bin_PROGRAMS = psphot psphotForced psphotFullForce psphotMinimal psphotMakePSF psphotStack psphotModelTest psmakecff
+bin_PROGRAMS = psphot psphotForced psphotFullForce psphotFullForceSummary psphotMinimal psphotMakePSF psphotStack psphotModelTest psmakecff
 # bin_PROGRAMS = psphotPetrosianStudy psphotTest psphotMomentsStudy 
 
@@ -39 +39 @@
 psphotFullForce_LDFLAGS = $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 psphotFullForce_LDADD = libpsphot.la
+
+psphotFullForceSummary_CFLAGS = $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
+psphotFullForceSummary_LDFLAGS = $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
+psphotFullForceSummary_LDADD = libpsphot.la
 
 psphotMinimal_CFLAGS = $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
@@ -99 +103 @@
         psphotMosaicChip.c         \
         psphotCleanup.c
+
+# combine full force results from several inputs
+psphotFullForceSummary_SOURCES = \
+        psphotFullForceSummary.c \
+        psphotFullForceSummaryReadout.c
 
 # forced photometry of specified positions given a specified psf

branches/eam_branches/ipp-20131211/psphot/src/psmakecff.c

@@ -2 +2 @@
 # include <config.h>
 # endif
+
+// psmakecff : A program to make read a cmf file and write a cff file
+// The real work is done in psModules.
 
 #include <stdio.h>
@@ -8 +11 @@
 #include "psphot.h"
 
+// For simplicilty, this program's (simple) functions are all contained in this file.
 static pmConfig* psmakecffArguments(int, char**);
 static bool psmakecffParseCamera(pmConfig *);
@@ -25 +29 @@
 //    psphotVersionPrint();
 
-    // load input data (config and images (signal, noise, mask)
     if (!psmakecffParseCamera (config)) {
         psErrorStackPrint(stderr, "Error setting up the camera\n");
@@ -31 +34 @@
     }
 
-    // call psphot for each readout
     if (!psmakecffImageLoop (config)) {
         psErrorStackPrint(stderr, "Error in the psphot image loop\n");
@@ -37 +39 @@
     }
 
-//    psLogMsg ("psphot", PS_LOG_WARN, "complete psphot run: %f sec\n", psTimerMark ("complete"));
-
-//    psErrorCode exit_status = psphotGetExitStatus();
-//    psphotCleanup (config);
     exit (0);
 }
@@ -73 +71 @@
     return config;
 }
+
 static bool psmakecffParseCamera(pmConfig *config) {
     bool status = false;
@@ -127 +126 @@
     psAssert (recipe, "missing recipe?");
 
-//    psImageMaskType maskTest = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT");
 
     // for psphot, we force data to be read at the chip level
     while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) {
-        psLogMsg ("psmakecmf", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
+        psLogMsg ("psmakecff", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
         if (! chip->process || ! chip->file_exists) { continue; }
 
-#ifdef notdef
-        pmFPAfileActivate (config->files, false, NULL);
-        pmFPAfileActivate (config->files, true, "PSPHOT.LOAD");
-#endif
         if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psmakecff.");
 
         // there is now only a single chip (multiple readouts?). loop over it and process
         while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) {
-            psLogMsg ("psmakecmf", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+            psLogMsg ("psmakecff", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
 
             // process each of the readouts
             while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) {
-                psLogMsg ("psmakecmf", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+                psLogMsg ("psmakecff", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
                 if (! readout->data_exists) { continue; }
 
                 pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell);
                 if (hdu && hdu != lastHDU) {
+                    // XXX: probably should do this
                     // psphotVersionHeaderFull(hdu->header);
                     lastHDU = hdu;
@@ -157 +152 @@
 
         }
-        // Defer output and closing of files until we've (possibly) done the NFrames analysis below
-//        pmFPAfileActivate (config->files, false, NULL);
-
         if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for Chip in psmakecff.");
     }
     if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for FPA in psphot.");
 
-    // fail if we failed to handle an error
+    // fail if we encountered an unhandled error
     if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!");
 

branches/eam_branches/ipp-20131211/psphot/src/psphot.h

@@ -361 +361 @@
 bool psphotModelTestReadout(pmConfig *config, const pmFPAview *view, const char *filerule);
 
+bool psphotFullForceSummaryReadout (pmConfig * config, const pmFPAview *view);
+
 int psphotFileruleCount(const pmConfig *config, const char *filerule);
 bool psphotFileruleCountSet(const pmConfig *config, const char *filerule, int num);

branches/eam_branches/ipp-20131211/psphot/src/psphotChooseAnalysisOptions.c

@@ -60 +60 @@
 }
 
-bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos);
+static bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos);
 
 // this function use an internal flag to mark sources which have already been measured
@@ -121 +121 @@
     psSphereRot *toGal = NULL;
     float GAL_LIMIT = 0.0;
+    float GAL_LIMIT_BULGE = 0.0;
+    float GAL_LIMIT_SIGMA2 = 0.0;
     bool useGAL_LIMIT = psMetadataLookupBool(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.USE");
     assert (status);
     if (useGAL_LIMIT) {
+        toGal = psSphereRotICRSToGalactic();
         GAL_LIMIT = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT");
-        toGal = psSphereRotICRSToGalactic();
+        assert (status);
+        GAL_LIMIT = DEG_TO_RAD(GAL_LIMIT);
+        GAL_LIMIT_BULGE = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE");
+        assert (status);
+        GAL_LIMIT_BULGE = DEG_TO_RAD(GAL_LIMIT_BULGE);
+        float GAL_LIMIT_SIGMA = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE.SIGMA");
+        assert (status);
+        assert(GAL_LIMIT_SIGMA > 0);
+        GAL_LIMIT_SIGMA = DEG_TO_RAD(GAL_LIMIT_SIGMA);
+        GAL_LIMIT_SIGMA2 = GAL_LIMIT_SIGMA * GAL_LIMIT_SIGMA;
     }
 
@@ -135 +147 @@
     }
 
+    float petroFluxLim = NAN;
     float extFitFluxLim = NAN;
 
@@ -140 +153 @@
         float extFitMagLimDefault = NAN;
         float extFitMagLim = NAN;
+        float petroMagLimDefault = NAN;
+        float petroMagLim = NAN;
 
         // match to the given filter
@@ -156 +171 @@
             // find a matching filter or default to 'any'
             if (!strcasecmp (thisFilter, "any")) {
-                psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT");
-                psAssert(extFitMagLimStr, "missing MAG.LIMIT");
+                psString petroMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO");
+                psAssert(petroMagLimStr, "missing MAG.LIMIT.PETRO");
+                petroMagLimDefault = atof (petroMagLimStr);
+
+                psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT");
+                psAssert(extFitMagLimStr, "missing MAG.LIMIT.EXTFIT");
                 extFitMagLimDefault = atof (extFitMagLimStr);
             }
 
-            // find a matching filter or default to 'any'
             if (!strcasecmp (thisFilter, filterID)) {
-                psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT");
-                psAssert(extFitMagLimStr, "missing MAG.LIMIT");
+                psString petroMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO");
+                psAssert(petroMagLimStr, "missing MAG.LIMIT.PETRO");
+                petroMagLim = atof (petroMagLimStr);
+
+                psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT");
+                psAssert(extFitMagLimStr, "missing MAG.LIMIT.EXTFIT");
                 extFitMagLim = atof (extFitMagLimStr);
                 break;
             }
         }
+        psFree(iter);
+        if (!isfinite (petroMagLim)) petroMagLim = petroMagLimDefault;
         if (!isfinite (extFitMagLim)) extFitMagLim = extFitMagLimDefault;
 
@@ -185 +209 @@
         psAssert (status, "missing FPA.ZP?");
 
+        petroFluxLim = exptime * pow (10.0, 0.4*(zeropt - petroMagLim));
         extFitFluxLim = exptime * pow (10.0, 0.4*(zeropt - extFitMagLim));
     }
@@ -225 +250 @@
             psSphere ptGal, ptSky;
             GetGalacticCoords (&ptGal, &ptSky, toGal, chip, source->peak->xf, source->peak->yf);
-            if (fabs(ptGal.d) < GAL_LIMIT) continue;
+            float l = ptGal.r;
+            float b_min = GAL_LIMIT + GAL_LIMIT_BULGE * exp(-0.5*(l*l/GAL_LIMIT_SIGMA2));
+            if (fabs(ptGal.d) < b_min) continue;
             // include an exception for low density skycells below the limit?
         }
 
         // for petro and extFit, we will either use the mag limits or the S/N
+        if (doPetrosian) {
+            if (isfinite(petroFluxLim)) {
+                if (source->moments->KronFlux > petroFluxLim) {
+                    source->tmpFlags |= PM_SOURCE_TMPF_PETRO;
+                }
+            } else if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) {
+                source->tmpFlags |= PM_SOURCE_TMPF_PETRO;
+            }
+        }
         if (isfinite(extFitFluxLim)) {
             if (source->moments->KronFlux > extFitFluxLim) {
                 source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT;
-                if (doPetrosian) source->tmpFlags |= PM_SOURCE_TMPF_PETRO;
             }
         } else {
             if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) {
                 source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT;
-                if (doPetrosian) source->tmpFlags |= PM_SOURCE_TMPF_PETRO;
             }
         }
@@ -244 +278 @@
 
     psLogMsg ("psphot.options", PS_LOG_WARN, "choose analysis options for %ld sources: %f sec\n", sources->n, psTimerMark ("psphot.options"));
+
+    psFree(toGal);
 
     return true;
@@ -287 +323 @@
     psSphereRot *toGal = NULL;
     float GAL_LIMIT = 0.0;
+    float GAL_LIMIT_BULGE = 0.0;
+    float GAL_LIMIT_SIGMA2 = 0.0;
     bool useGAL_LIMIT = psMetadataLookupBool(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.USE");
     assert (status);
     if (useGAL_LIMIT) {
+        toGal = psSphereRotICRSToGalactic();
         GAL_LIMIT = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT");
-        toGal = psSphereRotICRSToGalactic();
+        assert (status);
+        GAL_LIMIT = DEG_TO_RAD(GAL_LIMIT);
+        GAL_LIMIT_BULGE = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE");
+        assert (status);
+        GAL_LIMIT_BULGE = DEG_TO_RAD(GAL_LIMIT_BULGE);
+        float GAL_LIMIT_SIGMA = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE.SIGMA");
+        assert (status);
+        assert(GAL_LIMIT_SIGMA > 0);
+        GAL_LIMIT_SIGMA = DEG_TO_RAD(GAL_LIMIT_SIGMA);
+        GAL_LIMIT_SIGMA2 = GAL_LIMIT_SIGMA * GAL_LIMIT_SIGMA;
     }
 
@@ -365 +413 @@
     // find extFitFluxLim->data.F32[i] for i == image number
     psVector *extFitFluxLim = NULL;
+    psVector *petroFluxLim = NULL;
     if (magLimits) {
         extFitFluxLim = psVectorAlloc (inputFilters->n, PS_TYPE_F32);
+        petroFluxLim = psVectorAlloc (inputFilters->n, PS_TYPE_F32);
         psVectorInit (extFitFluxLim, NAN);
+        psVectorInit (petroFluxLim, NAN);
 
         float extFitMagLimDefault = NAN;
+        float petroMagLimDefault = NAN;
 
         // match mag limits (flux limits) to the filters
@@ -383 +435 @@
             // save the default value to assign to unset filters
             if (!strcasecmp (thisFilter, "any")) {
-                psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT");
-                psAssert(magString, "missing MAG.LIMIT");
+                psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT");
+                psAssert(magString, "missing MAG.LIMIT.EXTFIT");
                 extFitMagLimDefault = atof (magString);
+
+                magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO");
+                psAssert(magString, "missing MAG.LIMIT.PETRO");
+                petroMagLimDefault = atof (magString);
                 continue;
             }
@@ -393 +449 @@
                 if (i == chisqNum) continue;
                 if (!strcasecmp (thisFilter, inputFilters->data[i])) {
-                    psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT");
-                    psAssert(magString, "missing MAG.LIMIT");
+                    psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO");
+                    psAssert(magString, "missing MAG.LIMIT.PETRO");
                     float magvalue = atof (magString);
+                    petroFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - magvalue));
+
+                    magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT");
+                    psAssert(magString, "missing MAG.LIMIT.EXTFIT");
+                    magvalue = atof (magString);
                     extFitFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - magvalue));
                     break;
@@ -401 +462 @@
             }
         }
+        psFree(iter);
 
         for (int i = 0; i < num; i++) {
             if (i == chisqNum) continue;
-            if (isfinite(extFitFluxLim->data.F32[i])) continue;
-            extFitFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - extFitMagLimDefault));
+            if (!isfinite(petroFluxLim->data.F32[i])) {
+                petroFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - petroMagLimDefault));
+            }
+            if (!isfinite(extFitFluxLim->data.F32[i])) {
+                extFitFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - extFitMagLimDefault));
+            }
         }
     }
@@ -420 +486 @@
         bool doObjectRadial = false;
         bool doObjectExtFit = false;
-        for (int j = 0; !doObjectExtFit && !doObjectRadial && (j < object->sources->n); j++) {
+        bool doObjectPetrosian = false;
+        for (int j = 0; !doObjectExtFit && !doObjectRadial && !doObjectPetrosian && (j < object->sources->n); j++) {
 
             pmSource *source = object->sources->data[j];
@@ -451 +518 @@
             if (extFitAll) {
                 doObjectExtFit = true;
+                doObjectPetrosian = doPetrosian;
                 continue;
             }
@@ -465 +533 @@
                 psSphere ptGal, ptSky;
                 GetGalacticCoords (&ptGal, &ptSky, toGal, chips->data[imageID], source->peak->xf, source->peak->yf);
-                if (fabs(ptGal.d) < GAL_LIMIT) continue;
-                // include an exception for low density skycells below the limit?
-            }
-
-            float fluxLim = extFitFluxLim ? extFitFluxLim->data.F32[imageID] : NAN;
+                float l = ptGal.r;
+                float b_min = GAL_LIMIT + GAL_LIMIT_BULGE * exp(-0.5*(l*l/GAL_LIMIT_SIGMA2));
+                if (fabs(ptGal.d) < b_min) continue;
+            }
+
+            float fluxLim = NAN;
+            // for petro and extFit, we will either use the mag limits or the S/N
+            if (doPetrosian) {
+                fluxLim = petroFluxLim ? petroFluxLim->data.F32[imageID] : NAN;
+                if (isfinite(fluxLim)) {
+                    if (source->moments->KronFlux > extFitFluxLim->data.F32[imageID]) {
+                        doObjectPetrosian = true;
+                    }
+                } else {
+                    if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) {
+                        doObjectPetrosian = true;
+                    }
+                }
+            }
+            fluxLim = extFitFluxLim ? extFitFluxLim->data.F32[imageID] : NAN;
             // for petro and extFit, we will either use the mag limits or the S/N
             if (isfinite(fluxLim)) {
@@ -503 +586 @@
             if (source->modelPSF == NULL) continue;
                 
-            // Do the fits if the recipe requests we do extended source fits to everything
             if (doObjectExtFit) {
                 source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT;
-                if (doPetrosian) source->tmpFlags |= PM_SOURCE_TMPF_PETRO;
-            }
+            }
+            if (doObjectPetrosian) {
+                source->tmpFlags |= PM_SOURCE_TMPF_PETRO;
+            }
 
             // Do the fits if the recipe requests we do extended source fits to everything
@@ -522 +606 @@
     psLogMsg ("psphot.options", PS_LOG_WARN, "choose analysis options for %ld objects: %f sec\n", objects->n, psTimerMark ("psphot.options"));
 
+    psFree(toGal);
+
     return true;
 }
 
-bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos) {
+static bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos) {
 
     pmFPA *fpa = chip->parent;
@@ -543 +629 @@
     psSphereRotApply (ptGal, toGal, ptSky);
 
+    // psSphereRotApply insures that 0 < r < 2PI. We want -PI < b <= PI
+    if (ptGal->r > M_PI) {
+        ptGal->r -= 2.0 * M_PI;
+    }
+
     return true;
 

branches/eam_branches/ipp-20131211/psphot/src/psphotGalaxyShape.c

@@ -283 +283 @@
     }
 
+#ifdef SAVE_BEST_MODEL
+    // Save model with smallest chisq
     if (isfinite(chisqBest)) {
+#else 
+    // Save model with nominal parameters
+    {
+        fRmajorBest = 1;
+        fRminorBest = 1;
+#endif
         // now save the best fitting model as the source's extended model
         psEllipseAxes testAxes = guessAxes;
@@ -293 +301 @@
         psphotGalaxyShapeSource (pcm, source, maskVal, psfSize, false);
 
+        // Replace modelEXT with this model
+        // XXX: only do this if the model is good
         psFree (source->modelEXT);
 
@@ -299 +309 @@
         source->mode |= PM_SOURCE_MODE_EXTMODEL;
         source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT;
-
-        // adjust the window so the subtraction covers the faint wings
-        // psphotSetRadiusMoments(&fitRadius, &windowRadius, readout, source, markVal);
 
         // cache the model flux

branches/eam_branches/ipp-20131211/pstamp/scripts/dqueryparse.pl

@@ -502 +502 @@
     $command .= " -need_magic" if $need_magic;
 
-    my $rlabel = "dq_ud_" . $label if $label;
+    my $rlabel = "ps_ud_" . $label if $label;
     $command .= " -rlabel $rlabel" if $rlabel;
 

branches/eam_branches/ipp-20131211/pstamp/scripts/pstamp_checkdependent.pl

@@ -27 +27 @@
 my $IPP_DIFF_MODE_STACK_STACK = 4;
 
-my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile);
+my ($dep_id, $stage, $stage_id, $component, $imagedb, $label, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile);
 my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update);
 
@@ -36 +36 @@
     'component=s'   =>  \$component,
     'imagedb=s'     =>  \$imagedb,      # dbname for images lookups.
-    'rlabel=s'      =>  \$rlabel,
+    'label=s'       =>  \$label,        # request's label
+    'rlabel=s'      =>  \$rlabel,       # pstampDependent.rlabel (deprecated)
     'need_magic'    =>  \$need_magic,
     'fault_count=i' =>  \$fault_count,
@@ -73 +74 @@
 }
 
+if ($label) {
+    # rlabel is deprecated. Use one based on the supplied label parameter which is the current label 
+    # for the request, which may be different than the one given to the dependent when the job was parsed.
+    # XXX: having the convention that update label is 'ps_ud_' . $label of request embedded here 
+    # (and formerly in pstampparse.pl) is not particularly clean but it's simple.
+    my $new_rlabel = 'ps_ud_' . $label;
+    if ($new_rlabel ne $rlabel) {
+        print "Notice: using $new_rlabel instead of $rlabel for update label.\n";
+        $rlabel = $new_rlabel;
+    }
+}
+
+
 if (!$ps_dbserver) {
     $ps_dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
@@ -559 +573 @@
             }
             return $warp_status;
+        } elsif ($warp->{quality} != 0) {
+            print STDERR "input warp has quality error\n";
+            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
+            return $PSTAMP_GONE;
         }
         # warps are ready fall through and queue the diff update

branches/eam_branches/ipp-20131211/tools/mysql-dump/gpc1_install.sh

@@ -62 +62 @@
 log INFO "Symlinking backup file [$TARGET/$BACKUP_FILE] to distribution [$DISTRIBUTION_TARGET]"
 /usr/bin/ln -f $TARGET/$BACKUP_FILE $DISTRIBUTION_TARGET
-/usr/bin/mv -f $TARGET/$MD5FILE $DISTRIBUTION_MD5
+/usr/bin/cp -f $TARGET/$MD5FILE $DISTRIBUTION_MD5
 
 # if it is between 0 and 4, ingest into gpc1_0 or gpc1_1

branches/eam_branches/ipp-20131211/tools/neb_rawOTA_host_scan.pl

@@ -21 +21 @@
     ) or die "Unable to connect to database $DBI::errstr\n";
 
-my ($host,$min_ins_id,$verbose,$limit,$continue);
+my ($host,$min_ins_id,$verbose,$limit,$continue,$alt,$vol_label);
 $min_ins_id = 0;
 $verbose = 0;
 $limit = 10000;
 $continue = 0;
+$alt = 0;
+$vol_label = 0;
 
 GetOptions(
@@ -33 +35 @@
     'limit|l=s'     => \$limit,
     'continue|c'    => \$continue,
+    'alternate|a'   => \$alt,
+    'vol_label|L=s'   => \$vol_label,
     ) or pod2usage( 2 );
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
@@ -39 +43 @@
     defined($host);
 
-my $volume = $host . ".0";
+my $volume = $host . "." . $vol_label;
 my $rr;
 
@@ -50 +54 @@
 my $vol_id = shift( @{ ${ $r_vol }[0] });
 
+# Get max ins_id
+my $max_id_sth = "SELECT ins_id FROM instance WHERE vol_id = $vol_id ORDER BY ins_id DESC limit 1";
+my $r_max_id   = $db->selectall_arrayref( $max_id_sth );
+unless (defined(${ $r_max_id }[0])) {
+    die "Cannot find max_id";
+}
+my $max_ins_id = shift( @{ ${ $r_max_id }[0] });
+
 if ($verbose) {
-    print "$host $volume $vol_id $min_ins_id\n";
+    print "$host $volume $vol_id $min_ins_id $max_ins_id\n";
 }
 
@@ -70 +82 @@
 my $number = 0;
 do {
-    my $ins_id_sth = "SELECT ins_id,so_id,uri FROM instance WHERE vol_id = ${vol_id} AND ins_id > $min_ins_id LIMIT $limit";
+    my $ins_id_sth = "SELECT ins_id,so_id,uri FROM instance WHERE vol_id = ${vol_id} AND ins_id > $min_ins_id AND ins_id <= $max_ins_id  LIMIT $limit";
+    if ($alt) {
+        $ins_id_sth = "SELECT ins_id,so_id,uri,vol_id FROM instance WHERE ins_id > $min_ins_id AND ins_id <= $max_ins_id  LIMIT $limit";
+    }
     my $r_ins      = $db->selectall_arrayref( $ins_id_sth );
-    my $last_ins_id = 0;
+    my $last_ins_id = $max_ins_id;
     $number = 0;
     foreach $rr (@{ $r_ins }) {
-        my ($ins_id,$so_id,$uri) = @{ $rr };
+        my ($ins_id,$so_id,$uri,$v_id) = @{ $rr };
         if ($verbose) {
 #       print ("  $ins_id $so_id $uri\n");
@@ -81 +96 @@
         $number++;
         $last_ins_id = $ins_id;
+        if ($alt) {
+            if ($v_id != $vol_id) { next; }
+        }
         if ($uri !~ /ota...fits/) { next; }
         my $have_b_node = 0;
@@ -138 +156 @@
         $min_ins_id = $last_ins_id;
     }
-} while ($continue && $number != 0);
+} while (($continue && $number != 0) || ($min_ins_id < $max_ins_id));

@@ -47 +47 @@
 sctool
 releasetool
+fftool

@@ -26 +26 @@
 psphotFullForce
 psmakecff
+psphotFullForceSummary