Changeset 35236

Timestamp:

Mar 5, 2013, 2:24:05 PM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130207

Files:

• . (modified) (1 prop)
• Nebulous (modified) (1 prop)
• Nebulous-Server (modified) (1 prop)
• Nebulous-Server/bin/nebdiskd (modified) (1 diff)
• dbconfig/background.md (modified) (1 diff)
• dbconfig/cam.md (modified) (1 diff, 1 prop)
• dbconfig/changes.txt (modified) (6 diffs, 1 prop)
• dbconfig/config.md (modified) (1 diff)
• dbconfig/lap.md (modified) (1 diff)
• dbconfig/release.md (modified) (5 diffs)
• dbconfig/staticsky.md (modified) (1 diff)
• ippMonitor (modified) (1 prop)
• ippMonitor/Makefile.in (modified) (1 diff)
• ippMonitor/raw/czartool_getplot.php (modified) (1 diff)
• ippMonitor/raw/heathers_plot.php (copied) (copied from trunk/ippMonitor/raw/heathers_plot.php )
• ippMonitor/raw/ipp.imfiles.dat (modified) (1 diff)
• ippScripts/scripts/automate_stacks.pl (modified) (3 diffs)
• ippScripts/scripts/background_chip.pl (modified) (7 diffs)
• ippScripts/scripts/background_warp.pl (modified) (3 diffs)
• ippScripts/scripts/camera_exp.pl (modified) (1 diff)
• ippScripts/scripts/chip_imfile.pl (modified) (1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/diff_skycell.pl (modified) (1 diff)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/ipp_cleanup.pl (modified) (2 diffs)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (7 diffs, 1 prop)
• ippScripts/scripts/skycalibration.pl (modified) (1 diff)
• ippScripts/scripts/warp_skycell.pl (modified) (1 diff, 1 prop)
• ippTasks (modified) (1 prop)
• ippTasks/background.pro (modified) (3 diffs)
• ippTasks/burntool.pro (modified) (5 diffs)
• ippTasks/summit.copy.pro (modified) (1 diff)
• ippTools/configure.ac (modified) (2 diffs)
• ippTools/share (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (2 diffs)
• ippTools/share/bgtool_chip.sql (modified) (2 diffs)
• ippTools/share/bgtool_chipinputs.sql (modified) (1 diff)
• ippTools/share/bgtool_definechip.sql (modified) (1 diff)
• ippTools/share/bgtool_definewarp.sql (modified) (1 diff)
• ippTools/share/bgtool_tochip.sql (modified) (1 diff)
• ippTools/share/bgtool_towarp.sql (modified) (1 diff)
• ippTools/share/bgtool_warp.sql (modified) (1 diff)
• ippTools/share/bgtool_warpinputs.sql (modified) (1 diff)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/difftool_definewarpstack.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (9 diffs, 1 prop)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/share/releasetool_definerelexp.sql (modified) (1 diff)
• ippTools/share/releasetool_definerelgroup_select_data_group.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelgroup_select_data_group.sql )
• ippTools/share/releasetool_definerelgroup_select_exp_data_group.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelgroup_select_exp_data_group.sql )
• ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql )
• ippTools/share/releasetool_definerelgroup_select_lap.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelgroup_select_lap.sql )
• ippTools/share/releasetool_definerelstack.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelstack.sql )
• ippTools/share/releasetool_definerelstack_with_skycal.sql (copied) (copied from trunk/ippTools/share/releasetool_definerelstack_with_skycal.sql )
• ippTools/share/releasetool_listrelexp.sql (modified) (2 diffs)
• ippTools/share/releasetool_pendingrelgroup.sql (copied) (copied from trunk/ippTools/share/releasetool_pendingrelgroup.sql )
• ippTools/share/releasetool_tocalibexp.sql (copied) (copied from trunk/ippTools/share/releasetool_tocalibexp.sql )
• ippTools/share/reltool_pendinggroup.sql (copied) (copied from trunk/ippTools/share/reltool_pendinggroup.sql )
• ippTools/share/warptool_scmap.sql (modified) (1 diff)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/addtool.c (modified) (1 diff)
• ippTools/src/addtoolConfig.c (modified) (1 diff)
• ippTools/src/bgtool.c (modified) (18 diffs)
• ippTools/src/bgtoolConfig.c (modified) (4 diffs)
• ippTools/src/camtool.c (modified) (6 diffs, 1 prop)
• ippTools/src/camtoolConfig.c (modified) (3 diffs, 1 prop)
• ippTools/src/difftool.c (modified) (2 diffs)
• ippTools/src/disttool.c (modified) (12 diffs)
• ippTools/src/disttoolConfig.c (modified) (1 diff)
• ippTools/src/magictool.c (modified) (1 prop)
• ippTools/src/releasetool.c (modified) (16 diffs)
• ippTools/src/releasetool.h (modified) (1 diff)
• ippTools/src/releasetoolConfig.c (modified) (6 diffs)
• ippTools/src/staticskytool.c (modified) (2 diffs)
• ippTools/src/staticskytoolConfig.c (modified) (1 diff)
• ippconfig (modified) (1 prop)
• ippconfig/dvo.photcodes (modified) (4 diffs)
• ippconfig/gpc1/psphot.config (modified) (3 diffs, 1 prop)
• ippconfig/recipes/ppStats.config (modified) (1 diff)
• ippconfig/recipes/ppStatsFromMetadata.config (modified) (2 diffs)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/psphot.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)
• ppImage/src (modified) (1 prop)
• ppStack/src/ppStackCamera.c (modified) (4 diffs)
• ppStack/src/ppStackCombineAlternate.c (modified) (1 diff)
• ppStack/src/ppStackLoop.c (modified) (2 diffs)
• ppStack/src/ppStackOptions.c (modified) (1 diff)
• ppStack/src/ppStackOptions.h (modified) (1 diff)
• ppStack/src/ppStackSetup.c (modified) (1 diff)
• psLib/src/imageops/psImageInterpolate.c (modified) (1 diff)
• psModules (modified) (1 prop)
• psModules/src/imcombine/pmStack.c (modified) (1 diff)
• psModules/src/objects/models/pmModel_TRAIL.c (modified) (1 diff)
• psconfig (modified) (1 prop)
• psphot (modified) (1 prop)
• psphot/src (modified) (1 prop)
• psphot/src/psphotSourceSize.c (modified) (2 diffs)
• psphot/src/psphotStackImageLoop.c (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• psvideophot (modified) (1 prop)
• tools/fixburntool (modified) (1 diff)
• tools/heathers_plots (copied) (copied from trunk/tools/heathers_plots )
• tools/heathers_plots/alpha (copied) (copied from trunk/tools/heathers_plots/alpha )
• tools/heathers_plots/alpha/README (copied) (copied from trunk/tools/heathers_plots/alpha/README )
• tools/heathers_plots/alpha/grab_cam_LAP.sql (copied) (copied from trunk/tools/heathers_plots/alpha/grab_cam_LAP.sql )
• tools/heathers_plots/alpha/grab_skycal_LAP.sql (copied) (copied from trunk/tools/heathers_plots/alpha/grab_skycal_LAP.sql )
• tools/heathers_plots/alpha/mangle.LAP.cam.pl (copied) (copied from trunk/tools/heathers_plots/alpha/mangle.LAP.cam.pl )
• tools/heathers_plots/alpha/plot.cam.py (copied) (copied from trunk/tools/heathers_plots/alpha/plot.cam.py )
• tools/heathers_plots/alpha/plot.sky.py (copied) (copied from trunk/tools/heathers_plots/alpha/plot.sky.py )
• tools/heathers_plots/heathers_plots.py (copied) (copied from trunk/tools/heathers_plots/heathers_plots.py )

branches/eam_branches/ipp-20130207/Nebulous-Server/bin/nebdiskd

@@ -392 +392 @@
     my @data = split(/:/, $df_data);
     $df_info{"blocks"} = $data[1];
-    $df_info{"used"} = $data[2];
+    $df_info{"used"} = $data[1]-$data[3];
     $log->debug("blocks = " . $df_info{"blocks"} . " / used = " . $df_info{"used"});
     return \%df_info;

branches/eam_branches/ipp-20130207/dbconfig/background.md

@@ -5 +5 @@
     chip_bg_id          S64     0
     chip_id             S64     0
+    cam_id              S64     0
     state               STR     64
     workdir             STR     255

branches/eam_branches/ipp-20130207/dbconfig/cam.md

@@ -118 +118 @@
     deteff_uq      F32      0
     quality        S16      0
+    background_model  S16   0
 END
 

branches/eam_branches/ipp-20130207/dbconfig/changes.txt

@@ -2267 +2267 @@
 
 
+-- Update camProcessedExp to hold a background_model value
+
+ALTER TABLE camProcessedExp add column background_model SMALLINT after quality;
+UPDATE dbversion set schema_version = '1.1.74', updated= CURRENT_TIMESTAMP();
+
+
+-- Note the following 3 changes were implemented in the ipp-20121218 tag
+-- without changing the schema_version from 1.1.74
+
+-- Add cam_id to chipBackgroundRun
+ALTER TABLE chipBackgroundRun ADD COLUMN cam_id BIGINT DEFAULT 0 AFTER chip_id;
+
+-- add fwhm measuements to skycalResult
+ALTER TABLE skycalResult ADD COLUMN fwhm_major FLOAT AFTER zpt_stdev;
+ALTER TABLE skycalResult ADD COLUMN fwhm_minor FLOAT AFTER fwhm_major;
+
 -- new tables for managing the "release" of ipp data
 
 CREATE TABLE survey (
-    surveyID    INT(8),
+    surveyID    INT,
     surveyName  VARCHAR(16) NOT NULL,
     description VARCHAR(255),
@@ -2277 +2293 @@
 ) ENGINE=InnoDB CHARSET=latin1;
 
-
 CREATE TABLE ippRelease (
-    rel_id      INT(8) AUTO_INCREMENT,
-    surveyID    INT(8),
-    releaseName VARCHAR(64),
-    release_state VARCHAR(16),    -- active, pending archive drop 
-    dataRelease INT(8),         -- PSPS dataRelease
+    rel_id      INT AUTO_INCREMENT,
+    surveyID    INT,
+    release_name VARCHAR(64),
+    release_state VARCHAR(16),   -- active, pending, archive, drop 
+    dataRelease INT,         -- PSPS dataRelease
+    priority    INT,
     PRIMARY KEY(rel_id),
-    KEY(releaseName),
-    KEY(state),
+    KEY(release_name),
+    KEY(release_state),
+    KEY(priority),
     FOREIGN KEY(surveyID) REFERENCES survey(surveyID)
 ) ENGINE=Innodb DEFAULT CHARSET=latin1;
@@ -2292 +2309 @@
 CREATE TABLE relExp (
     relexp_id   BIGINT AUTO_INCREMENT,
-    rel_id      INT(8),
+    rel_id      INT,
     exp_id      BIGINT,
     chip_id     BIGINT,         -- links to the runs that supplied the data
     cam_id      BIGINT,         -- for this release
+    group_id    INT,         -- id of relGroup that this exposure is contained in
     state       VARCHAR(16),    -- released, pending, archive, drop  
-    flags       INT UNSIGNED,   -- flags for relphot, relastro, ??
+    flags       INT UNSIGNED,  -- flags for relphot, relastro, ??
     zpt_obs     FLOAT,          -- calibrated zero point for this release of
     zpt_stdev   FLOAT,          -- this exposure
     path_base   VARCHAR(255),   -- path_base of any supporting files for this
                                 -- release of this exposure.
+    fault SMALLINT NOT NULL,
     registered  DATETIME,       -- insertion time for this row
     time_stamp  DATETIME,       -- time of last update for this row
@@ -2307 +2326 @@
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
     KEY (state),
+    KEY (fault),
     FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id),
     FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id),
@@ -2314 +2334 @@
 CREATE TABLE relStack (
     relstack_id BIGINT AUTO_INCREMENT,
-    rel_id      INT(8),
+    rel_id      INT,
+    stack_id    BIGINT,         -- id of the stackRun
     skycal_id   BIGINT,         -- id of the sky calibration run that supplied the calibration
-                                -- for this release. We get back to stack from here
-    skycell_id  varchar(64),
-    tess_id     varchar(64),
+                                -- for this release of this skycell.
+    skycell_id  VARCHAR(64),
+    tess_id     VARCHAR(64),
+    filter      VARCHAR(16),
     state       VARCHAR(16),    -- released, pending, archive, drop  
     flags       INT UNSIGNED,
@@ -2324 +2346 @@
     zpt_obs     FLOAT,          -- calibrated zero point for this release of this skycell
     zpt_stdev   FLOAT,
-    fwhm_major  FLOAT,          -- seeing as measured by staticsky
+    mjd_obs     INT UNSIGNED,   -- for nightly stacks the mjd_obs, zero for deep and reference
     path_base   VARCHAR(255),   -- path_base of any supporting files,
+    fault SMALLINT NOT NULL,
     registered  DATETIME,
     time_stamp  DATETIME,
     PRIMARY KEY (relstack_id),
     KEY (tess_id, skycell_id),
-    KEY (rel_id, tess_id, skycell_id),
+    KEY (rel_id, tess_id, skycell_id, filter, mjd_obs),
+    KEY (stack_type),
+    KEY (mjd_obs),
+    KEY (state),
+    KEY (fault),
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
-    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
-) ENGINE=innodb DEFAULT CHARSET=latin1;
-
-UPDATE dbversion set schema_version = '1.1.74', updated= CURRENT_TIMESTAMP();
+    FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE relGroup (
+    group_id    INT AUTO_INCREMENT,
+    rel_id      INT,
+    group_type  VARCHAR(16),
+    lap_id      BIGINT,
+    group_name  VARCHAR(16),
+    state       VARCHAR(16),    -- new full
+    label       VARCHAR(64),
+    exp_list_path   VARCHAR(255),
+    fault SMALLINT NOT NULL,
+    registered  DATETIME,
+    PRIMARY KEY (group_id),
+    KEY (rel_id, group_name),
+    KEY (group_type),
+    KEY (lap_id),
+    KEY (group_name),
+    KEY (state),
+    KEY (label),
+    KEY (fault),
+    FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- new table grouping multi filter lap runs into a group. This is used to
+-- determine when all 5 filters for a projection cell have completed processing
+CREATE TABLE lapGroup  (
+    seq_id  BIGINT,
+    projection_cell VARCHAR(64),
+    state   VARCHAR(16),
+    fault   SMALLINT,
+    PRIMARY KEY(seq_id, projection_cell),
+    KEY(state),
+    KEY(fault),
+    FOREIGN KEY(seq_id) REFERENCES lapSequence(seq_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+UPDATE dbversion set schema_version = '1.1.75', updated= CURRENT_TIMESTAMP();
+

branches/eam_branches/ipp-20130207/dbconfig/config.md

@@ -2 +2 @@
     pkg_name        STR     ippdb
     pkg_namespace   STR     ippdb
-    pkg_version     STR     1.1.71717171717171
+    pkg_version     STR     1.1.75
 END

branches/eam_branches/ipp-20130207/dbconfig/lap.md

@@ -30 +30 @@
     data_state     STR         64   # Key
 END
+
+lapGroup METADATA
+    seq_id          S64 0
+    projection_cell STR 64
+    state           STR 64
+    fault           S16 0
+end

branches/eam_branches/ipp-20130207/dbconfig/release.md

@@ -8 +8 @@
     rel_id      S32     0   # Primary Key AUTO_INCREMENT
     surveyID    S32     0   # fkey(surveyID) ref survey(surveyID)
-    releaseName STR     64
+    release_name STR     64
     release_state STR   16
     dataRelease S32     0
@@ -20 +20 @@
     chip_id     S64     0
     cam_id      S64     0
+    group_id    S32     0
     state       STR     16
     flags       U32     0
@@ -25 +26 @@
     zpt_stdev   F32     0
     path_base   STR     255
+    fault       S16     0
     registered  TAI     NULL
     time_stamp  UTC     NULL
@@ -32 +34 @@
     relstack_id S64     0   # Primary Key AUTO_INCREMENT
     rel_id      S32     0   # fkey(rel_id) ref ippRelease(rel_id)
-    skycal_id   S64     0   # fkey(skycal_id) ref skycalRun(skycal_id)
+    stack_id    S64     0   # fkey(stack_id) ref stackRun(stack_id)
+    skycal_id   S64     0 
     skycell_id  STR     64
     tess_id     STR     64
+    filter      STR     64
     state       STR     16
     flags       U32     0
@@ -40 +44 @@
     zpt_obs     F32     0
     zpt_stdev   F32     0
-    fwhm_major  F32     0
+    mjd_obs     U32     0
     path_base   STR     255
+    fault       S16     0
     registered  TAI     NULL
     time_stamp  TAI     NULL
 END
+
+relGroup METADATA
+    group_id    S32     0   # Primary Key AUTO_INCREMENT
+    rel_id      S32     0
+    group_type  STR     16
+    lap_id      S64     0
+    group_name  STR     16
+    state       STR     16
+    label       STR     64
+    exp_list_path STR   255
+    fault       S16     0
+    registered  TAI     NULL
+END

branches/eam_branches/ipp-20130207/dbconfig/staticsky.md

@@ -59 +59 @@
     zpt_obs        F32    0.0
     zpt_stdev      F32    0.0
+    fwhm_major     F32    0.0
+    fwhm_minor     F32    0.0
     quality        S16    0       # Key
     software_ver   STR    16

branches/eam_branches/ipp-20130207/ippMonitor/Makefile.in

@@ -76 +76 @@
 $(DESTWWW)/diskUsage.php \
 $(DESTWWW)/version.php \
-$(DESTWWW)/show_log.php 
+$(DESTWWW)/show_log.php \
+$(DESTWWW)/heathers_plot.php 
 
 DEFSRC = \

branches/eam_branches/ipp-20130207/ippMonitor/raw/czartool_getplot.php

@@ -34 +34 @@
 else if ($type=="id")
 $filePath = "$path/ippToPsps_density_".$label.".png";
+else if ($type=="hp_c")
+$filePath = "$path/heathers_plots/cam.png";
+else if ($type=="hp_s")
+$filePath = "$path/heathers_plots/skycal.png";
 
 

branches/eam_branches/ipp-20130207/ippMonitor/raw/ipp.imfiles.dat

@@ -24 +24 @@
 menutop   | menutop      | link    | czartool - ipptopsps         | czartool_ipptopsps.php
 menutop   | menutop      | link    | mask stats                   | maskStats.php
+menutop   | menutop      | link    | Heather's Plots              | heathers_plot.php
 
 menutop   | menutop      | plain   |                         | 

branches/eam_branches/ipp-20130207/ippScripts/scripts/automate_stacks.pl

@@ -117 +117 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-          -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks",
+          -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks --dbname",
           -exitval => 3,
           ) unless
@@ -124 +124 @@
     defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or
     defined $test_mode or defined $clean_old or defined $check_mode or
-    defined $confirm_stacks or defined $burntool_stats;
+    defined $confirm_stacks or defined $burntool_stats or defined $dbname;
 
 # Configurable parameters from our config file.
@@ -282 +282 @@
 unless(defined($camera)) {
     $camera = 'GPC1';
-}
-unless(defined($dbname)) {
-    $dbname = 'gpc1';
 }
 

branches/eam_branches/ipp-20130207/ippScripts/scripts/background_chip.pl

@@ -43 +43 @@
 # Parse the command-line arguments
 my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose,
-     $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $magicked );
+     $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $cam_path_base, $magicked );
 GetOptions(
     'chip_bg_id=s'      => \$chip_bg_id,    # chipBackgroundRun identifier
@@ -53 +53 @@
     'threads=s'         => \$threads,   # Number of threads to use
     'chip_path_base=s'  => \$chip_path_base, # optional chip_path_base
+    'cam_path_base=s'   => \$cam_path_base, # optional camera stage path_base
     'magicked=s'        => \$magicked,  # magicked status of input
     'verbose'           => \$verbose,   # Print to stdout
@@ -94 +95 @@
 my $dump_config;
 if ($chip_path_base and $no_update) {
+    # XXX: this path through the code has not been excercized in awhile and probably does not work
+
     # we are running outside of a chip_bg_run (perhaps by the postage stamp server) don't dump config
     # or do stats. Get path to input from command line.
@@ -119 +122 @@
     my $input = $$inputs[0];    # Input of interest
     $in_path = $input->{path_base};
+    $cam_path_base = $input->{cam_path_base};
     $magicked = $input->{magicked};
 }
 
 my $in_image = $ipprc->filename("PPIMAGE.CHIP", $in_path, $class_id);
-my $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id);
+my $in_mask;
+if ($cam_path_base) {
+    $in_mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
+} else {
+    $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id);
+}
 my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id);
 my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id);
@@ -177 +186 @@
 
 my $apply_auxiliary_mask = 1;
+# XXX Temporary lookup of auxiliary mask file from static location.
+# At some point I'll convert this to register the masks in the database and use detselect
 if ($apply_auxiliary_mask) {
     my @auxmask_start_date = qw(
@@ -238 +249 @@
     print "Auxililary mask file is $auxmask\n";
 }
-    
 
 
@@ -272 +282 @@
     &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats);
 
-    my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_CHIP";
+    my $resolved_stats = $ipprc->file_resolve($out_stats);
+    my $command = "$ppStatsFromMetadata $resolved_stats - BACKGROUND_CHIP";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);

branches/eam_branches/ipp-20130207/ippScripts/scripts/background_warp.pl

@@ -119 +119 @@
 my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id";
 
-# XXX: get this from recipe
-my $alternate_astrometry_label = 'M31.test.20130206.bgsub';
-# XXX: get exp_id from command line
-my $exp_id = 196758;
-
 if (!defined $imageName) {
     # go find our inputs
-    my $astrom_path_base;
-    if (defined $alternate_astrometry_label) {
-        my $command = "$camtool -processedexp -exp_id $exp_id -label $alternate_astrometry_label";
-        $command .= " -dbname $dbname" if $dbname;
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform camtool -processedexp: $error_code", $warp_bg_id, $error_code);
-        }
-        my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or
-            &my_die("Unable to parse metadata config doc", $warp_bg_id, $PS_EXIT_PROG_ERROR);
-        my $entry = $entries->[0];
-        $astrom_path_base = $entry->{path_base};
-    }
-
     my ($imageFile, $maskFile, $weightFile);
     ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX",  UNLINK => !$save_temps);
@@ -174 +153 @@
         $magicked = $file->{magicked};
 
-        if (!$astrom_path_base) {
-            my $cam_path = $file->{cam_path_base};
-            my $astrom = $ipprc->filename($astromSource, $cam_path);
-            &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry;
-            $astrometry = $astrom;
-        } elsif (!$astrometry) {
-            # using alternate astrometry
-            $astrometry = $ipprc->filename($astromSource, $astrom_path_base);
-        }
+        my $cam_path = $file->{cam_path_base};
+        my $astrom = $ipprc->filename($astromSource, $cam_path);
+        &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry;
+        $astrometry = $astrom;
     }
     close $imageFile;
@@ -238 +212 @@
     &my_die("Couldn't find expected output file: $out_stats", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats);
 
-    my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_WARP";
+    my $real_stats = $ipprc->file_resolve($out_stats);
+
+    my $command = "$ppStatsFromMetadata $real_stats - BACKGROUND_WARP";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);

branches/eam_branches/ipp-20130207/ippScripts/scripts/camera_exp.pl

@@ -379 +379 @@
             check_output($bkgModel, $replicateOutputs);
         }
+        $cmdflags = $cmdflags . " -background_model 1 ";
     }
 }

branches/eam_branches/ipp-20130207/ippScripts/scripts/diff_skycell.pl

@@ -299 +299 @@
 my $jpeg1Name = prepare_output("PPSUB.OUTPUT.JPEG1", $outroot, 1);
 my $jpeg2Name = prepare_output("PPSUB.OUTPUT.JPEG2", $outroot, 1);
+
+# do we always need to prepare this file?
+if ($reduction ne 'NOCONVDIFF') {
+    my $refConv = prepare_output("PPSUB.REF.CONV", $outroot, 1);
+}
 
 my ($inverseName, $inverseMask, $inverseVariance, $inverseSources);

branches/eam_branches/ipp-20130207/ippScripts/scripts/ipp_cleanup.pl

@@ -1957 +1957 @@
             addFilename (\@files, "PPBACKGROUND.OUTPUT", $path_base, $class_id);
             addFilename (\@files, "PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id);
+            addFilename (\@files, "PPBACKGROUND.OUTPUT.VARIANCE", $path_base, $class_id);
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode
@@ -2100 +2101 @@
             addFilename (\@files, "PSWARP.OUTPUT", $path_base, $skycell_id);
             addFilename (\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
+            addFilename (\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode

branches/eam_branches/ipp-20130207/ippScripts/scripts/publish_file.pl

@@ -92 +92 @@
 
 my $comment;                    # Comment for exposure
+my $exp_name_1;                    # Name of exposure 1
+my $exp_name_2;                    # Name of exposure 2
 if ($stage eq 'camera') {
     my $command =  "camtool -processedexp -cam_id $stage_id";
@@ -168 +170 @@
     my %negative;               # Data for negative diff detections
     foreach my $comp ( @$components ) {
-        my $path_base = $comp->{path_base}; # Base name for file
+        my $path_base = $comp->{path_base}; # Base name for file
         if (!$need_magic and $comp->{magicked}) {
             # This client is authorized to receive uncensored detections
@@ -178 +180 @@
         print "Warning: mis-matched comments\n" if defined $comment and $comment ne $comp->{comment};
         $comment = $comp->{comment} unless defined $comment;
+        $exp_name_1 = $comp->{exp_name_1} unless defined $exp_name_1;
+        $exp_name_2 = $comp->{exp_name_2} unless defined $exp_name_2;
 
         (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time};
@@ -192 +196 @@
         &my_die("input file does not exist: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$ipprc->file_exists($filename);
         my $resolved = $ipprc->file_resolve($filename);
-        
+
         &my_die("unable to resolve input file: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$resolved;
 
         $filename = $resolved;
 
-        my $cam_id = $comp->{cam_id_1};
-        #print "Getting info from camera stage $cam_id\n";
-        my $cam_command =  "$camtool -processedexp -cam_id $cam_id";
-        $cam_command .= " -dbname $dbname" if defined $dbname;
-        my ( $cam_success, $cam_error_code, $cam_full_buf, $cam_stdout_buf, $cam_stderr_buf ) =
-            run(command => $cam_command, verbose => $verbose);
-        &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $cam_success;
-        my $cam_metadata = $mdcParser->parse(join "", @$cam_stdout_buf) or
-            &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
-        my $cam_components = parse_md_list($cam_metadata) or
-            &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
-        &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$cam_components > 1;
-        my $cam_comp = $$cam_components[0];
-
-        my $exp_id = $comp->{exp_id_1};
-        #print "Getting info from raw stage $exp_id\n";
-        my $reg_command =  "$regtool -processedimfile -exp_id $exp_id";
-        $reg_command .= " -dbname $dbname" if defined $dbname;
-        my ( $reg_success, $reg_error_code, $reg_full_buf, $reg_stdout_buf, $reg_stderr_buf ) =
-            run(command => $reg_command, verbose => $verbose);
-        &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $reg_success;
-        my $reg_metadata = $mdcParser->parse(join "", @$reg_stdout_buf) or
-            &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
-        my $reg_components = parse_md_list($reg_metadata) or
-            &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
-        my $reg_comp = $$reg_components[0];
-
-        # Now get the difftype value from diff->{diff_mode}
-        my $difftype = "";
-        if ($comp->{diff_mode} == 1) {
-            $difftype = "WW";
-        } elsif ($comp->{diff_mode} == 2) {
-            $difftype = "WS";
-        } elsif ($comp->{diff_mode} == 3) {
-            $difftype = "SW"; # Not used yet
-        } elsif ($comp->{diff_mode} == 4) {
-            $difftype = "SS";
-        } else {
-            $difftype = "Unsupported diff_mode value: [" . $comp->{diff_mode} . "]";
-        }
+        my $cam_id = $comp->{cam_id_1};
+        #print "Getting info from camera stage $cam_id\n";
+        my $cam_command =  "$camtool -processedexp -cam_id $cam_id";
+        $cam_command .= " -dbname $dbname" if defined $dbname;
+        my ( $cam_success, $cam_error_code, $cam_full_buf, $cam_stdout_buf, $cam_stderr_buf ) =
+            run(command => $cam_command, verbose => $verbose);
+        &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $cam_success;
+        my $cam_metadata = $mdcParser->parse(join "", @$cam_stdout_buf) or
+            &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
+        my $cam_components = parse_md_list($cam_metadata) or
+            &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
+        &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$cam_components > 1;
+        my $cam_comp = $$cam_components[0];
+
+        my $exp_id = $comp->{exp_id_1};
+        #print "Getting info from raw stage $exp_id\n";
+        my $reg_command =  "$regtool -processedimfile -exp_id $exp_id";
+        $reg_command .= " -dbname $dbname" if defined $dbname;
+        my ( $reg_success, $reg_error_code, $reg_full_buf, $reg_stdout_buf, $reg_stderr_buf ) =
+            run(command => $reg_command, verbose => $verbose);
+        &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $reg_success;
+        my $reg_metadata = $mdcParser->parse(join "", @$reg_stdout_buf) or
+            &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
+        my $reg_components = parse_md_list($reg_metadata) or
+            &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
+        my $reg_comp = $$reg_components[0];
+
+        # Now get the difftype value from diff->{diff_mode}
+        my $difftype = "";
+        if ($comp->{diff_mode} == 1) {
+            $difftype = "WW";
+        } elsif ($comp->{diff_mode} == 2) {
+            $difftype = "WS";
+        } elsif ($comp->{diff_mode} == 3) {
+            $difftype = "SW"; # Not used yet
+        } elsif ($comp->{diff_mode} == 4) {
+            $difftype = "SS";
+        } else {
+            $difftype = "Unsupported diff_mode value: [" . $comp->{diff_mode} . "]";
+        }
 
         my $data = { zp => $zp,
@@ -250 +254 @@
                      output_format => $comp->{output_format},
                      direction => 1,
-                     comment => $reg_comp->{comment},
-                     obsmode => $reg_comp->{obs_mode},
-                     difftype => $difftype,
-                     sky => $cam_comp->{bg},
-                     shutoutc => $reg_comp->{dateobs},
+                     comment => $reg_comp->{comment},
+                     obsmode => $reg_comp->{obs_mode},
+                     difftype => $difftype,
+                     sky => $cam_comp->{bg},
+                     shutoutc => $reg_comp->{dateobs},
         };
 
-        #warn("Checking for positive");
+        #warn("Checking for positive");
         diff_check(\%positive, $data, "positive");
 
@@ -288 +292 @@
                          diff_id => $comp->{diff_id},
                          direction => 0,
-                         comment => $reg_comp->{comment},
-                         obsmode => $reg_comp->{obs_mode},
-                         difftype => $difftype,
-                         sky => $cam_comp->{bg},
-                         shutoutc => $reg_comp->{dateobs},
+                         comment => $reg_comp->{comment},
+                         obsmode => $reg_comp->{obs_mode},
+                         difftype => $difftype,
+                         sky => $cam_comp->{bg},
+                         shutoutc => $reg_comp->{dateobs},
             };
 
-            #warn("Checking for negative");
+            #warn("Checking for negative");
             diff_check(\%negative, $data, "negative");
 
@@ -326 +330 @@
     my $command = "$dsreg --add pub.$pub_id.$stage.$stage_id --copy --abspath --product $product --type $dsType --list $dsFileName";
     $command .= " --ps0 \"$comment\"" if defined $comment;
+    $command .= " --ps1 \"$exp_name_1\"" if defined $exp_name_1;
+    $command .= " --ps2 \"$exp_name_2\"" if defined $exp_name_2;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

branches/eam_branches/ipp-20130207/ippScripts/scripts/skycalibration.pl

@@ -231 +231 @@
     $command .= " -F PSASTRO.OUTPUT $output_sources_filerule";
     $command .= " -stats $stats";
-    $command .= " -recipe PPSTATS CAMSTATS";
+    # $command .= " -recipe PPSTATS CAMSTATS";
+    $command .= " -recipe PPSTATS SKYCALSTATS";
     $command .= " -dumpconfig $configuration" if $configuration;
     $command .= " -tracedest $traceDest";

branches/eam_branches/ipp-20130207/ippScripts/scripts/warp_skycell.pl

@@ -245 +245 @@
     print $weightFile "$weight\n";
     my $bkg;
-    if ($doBackground) {
-        $bkg    = $ipprc->filename("PSPHOT.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id});
+    if (($doBackground)) {
+        # &&($imfile->{background_model} == 1)
+        if ($imfile->{cam_background_model}) {
+            $bkg    = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{cam_path_base}, $imfile->{class_id});
+        }
+        else {
+            $bkg    = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id});
+        }
         print $bkgFile "$bkg\n";
     }

branches/eam_branches/ipp-20130207/ippTasks/background.pro

@@ -241 +241 @@
     else
       # save the DB name for the exit tasks
-      option $DB:$chip_advance_DB
+      option $DB:$bg_chip_advance_DB
       $run = $run -dbname $DB:$bg_chip_advance_DB
       $bg_chip_advance_DB ++
@@ -446 +446 @@
 
   periods      -poll $LOADPOLL
-#  periods      -exec $LOADEXEC
   periods      -exec 30
   periods      -timeout 60
@@ -461 +460 @@
     else
       # save the DB name for the exit tasks
-      option $DB:$warp_advance_DB
+      option $DB:$bg_warp_advance_DB
       $run = $run -dbname $DB:$bg_warp_advance_DB
       $bg_warp_advance_DB ++

branches/eam_branches/ipp-20130207/ippTasks/burntool.pro

@@ -1 +1 @@
 ## burntool.pro : -*- sh -*-
-
+## This task is designed to only be used when re-burntooling data.  It is not necessary for standard registration.
 check.globals
 
@@ -161 +161 @@
   periods         -exec $LOADEXEC
   periods         -timeout 30
+  active          false
   npending        1
 
@@ -216 +217 @@
   periods         -exec $LOADEXEC
   periods         -timeout 300
+  active          false
   npending        1
 
@@ -281 +283 @@
   periods         -exec $LOADEXEC
   periods         -timeout 30
+  active          false
   npending        1
 
@@ -336 +339 @@
   periods         -exec $RUNEXEC
   periods         -timeout 10800
+  active          false
   npending        30
   task.exec

branches/eam_branches/ipp-20130207/ippTasks/summit.copy.pro

@@ -6 +6 @@
 
 # pztool -adddatastore -inst isp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/skyprobe/index.txt
+# pztool -adddatastore -inst ssp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/ssp/index.txt                                                                                        
 # pztool -adddatastore -inst gpc1 -telescope ps1 -uri http://conductor/ds/gpc1/index.txt
 # pztool -adddatastore -inst allskycam -telescope ps1 -uri  http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt
+
 
 # NOTE: workdir / volume mangling and nebulous.  these tasks copy the

branches/eam_branches/ipp-20130207/ippTools/configure.ac

@@ -1 +1 @@
 AC_PREREQ(2.61)
 
-AC_INIT([ipptools], [1.1.73], [ipp-support@ifa.hawaii.edu])
+AC_INIT([ipptools], [1.1.75], [ipp-support@ifa.hawaii.edu])
 AC_CONFIG_SRCDIR([autogen.sh])
 
@@ -18 +18 @@
 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
-PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.71]) 
+PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.75]) 
 PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) 
 

branches/eam_branches/ipp-20130207/ippTools/share/Makefile.am

@@ -155 +155 @@
         difftool_completed_runs.sql \
         difftool_coalesce_run.sql \
+        difftool_definewarpstack.sql \
         difftool_definewarpstack_part1.sql \
         difftool_definewarpstack_part2.sql \
@@ -463 +464 @@
         sctool_list.sql \
         releasetool_listsurvey.sql \
+        releasetool_tocalibexp.sql \
         releasetool_listrelease.sql \
         releasetool_listrelexp.sql \
-        releasetool_definerelexp.sql
+        releasetool_definerelexp.sql \
+        releasetool_definerelstack.sql \
+        releasetool_definerelstack_with_skycal.sql \
+        releasetool_definerelgroup_select_lap.sql \
+        releasetool_definerelgroup_select_data_group.sql \
+        releasetool_definerelgroup_select_exp_data_group.sql \
+        releasetool_definerelgroup_select_exp_lap.sql \
+        releasetool_pendingrelgroup.sql
 

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chip.sql

@@ -5 +5 @@
     chipBackgroundRun.label,
     chipBackgroundRun.data_group,
+    chipBackgroundRun.chip_id,
+    chipBackgroundRun.cam_id,
     rawExp.exp_id,
     rawExp.exp_name,
@@ -12 +14 @@
     rawExp.ra,
     rawExp.decl,
-    rawExp.exp_time,
-    chipRun.chip_id
+    rawExp.exp_time
 FROM chipBackgroundRun
 JOIN chipBackgroundImfile USING(chip_bg_id)

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chipinputs.sql

@@ -1 +1 @@
 SELECT
-    chipProcessedImfile.*
+    chipProcessedImfile.*,
+    camProcessedExp.path_base AS cam_path_base
 FROM chipBackgroundRun
 JOIN chipRun USING(chip_id)
 JOIN chipProcessedImfile USING(chip_id)
+LEFT JOIN camProcessedExp ON chipBackgroundRun.cam_id = camProcessedExp.cam_id
 WHERE chipRun.state = 'full'
     AND chipProcessedImfile.fault = 0

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definechip.sql

@@ -1 +1 @@
 SELECT
-    chipRun.*
+    chipRun.*,
+    camRun.cam_id,
+    chip_bg_id,
+    CURRENT_TIMESTAMP AS registered
 FROM chipRun
 JOIN rawExp USING(exp_id)
-LEFT JOIN chipBackgroundRun USING(chip_id)
-WHERE chipRun.state = 'full'
+JOIN chipRun as altChipRun USING(exp_id)
+JOIN camRun ON altChipRun.chip_id = camRun.chip_id
+JOIN camProcessedExp USING(cam_id)
+LEFT JOIN chipBackgroundRun ON chipBackgroundRun.chip_id = chipRun.chip_id -- labelHook %s
+WHERE chipRun.state = 'full' AND camRun.state ='full' AND camProcessedExp.quality = 0

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definewarp.sql

@@ -1 +1 @@
 SELECT
-    warpRun.*,
-    chipBackgroundRun.chip_bg_id
+    warpRun.warp_id,
+    chipBackgroundRun.*
+    -- the following items are selected for aid in debugging
+    ,
+    warpsChipRun.chip_id as warpschip_id,
+    chipRun.chip_id AS thischip_id,
+    warpBackgroundRun.warp_bg_id
 FROM warpRun
 JOIN fakeRun USING(fake_id)
 JOIN camRun USING(cam_id)
-JOIN chipRun USING(chip_id)
-JOIN chipBackgroundRun USING(chip_id)
+JOIN chipRun as warpsChipRun ON camRun.chip_id = warpsChipRun.chip_id
 JOIN rawExp USING(exp_id)
-LEFT JOIN warpBackgroundRun USING(chip_bg_id)
+JOIN chipRun USING(exp_id)
+JOIN chipBackgroundRun ON chipRun.chip_id = chipBackgroundRun.chip_id 
+    AND (chipBackgroundRun.cam_id = camRun.cam_id OR chipBackgroundRun.cam_id = 0)
+LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.warp_bg_id -- label hook %s
 WHERE chipBackgroundRun.state = 'full'
-    AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed
+    AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- need warp to have completed so warpSkyCellMap is populated

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_tochip.sql

@@ -16 +16 @@
     AND (Label.active OR Label.active IS NULL)
 -- WHERE hook %s
-ORDER BY priority DESC, chip_bg_id
+ORDER BY priority DESC, chip_bg_id, class_id

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_towarp.sql

@@ -24 +24 @@
     AND (Label.active OR Label.active IS NULL)
 -- WHERE hook %s
-ORDER BY priority DESC, chip_bg_id
+ORDER BY priority DESC, warp_bg_id, skycell_id

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warp.sql

@@ -2 +2 @@
     warpBackgroundSkyfile.*,
     warpBackgroundRun.state,
+    warpBackgroundRun.warp_id,
+    camRun.cam_id,
+    warpBackgroundRun.chip_bg_id,
     warpBackgroundRun.workdir,
     warpBackgroundRun.label,
+    warpRun.label as warp_label,
     rawExp.exp_id,
     rawExp.exp_name,

branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warpinputs.sql

@@ -11 +11 @@
 JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
 JOIN fakeRun USING(fake_id)
-JOIN camProcessedExp USING(cam_id)
+JOIN camProcessedExp ON camProcessedExp.cam_id = fakeRun.cam_id
 WHERE warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed
     AND warpSkyfile.fault = 0

branches/eam_branches/ipp-20130207/ippTools/share/difftool_definewarpstack.sql

@@ -1 @@
--- Get list of warps that can be diffed, with any associated diff,
--- and the best stack to use as a template
--- Warps without an existing diff can be identified by a NULL diff_id
+-- Get list of warp-stack pairs that can be diffed
+-- and check results against existing diffs
 SELECT
-    warpsToDiff.warp_id,
-    warpsToDiff.skycell_id,
-    warpsToDiff.tess_id,
-    warpsToDiff.filter,
-    warpsToDiff.good_frac,
-    warpsToDiff.diff_id,
-    current_stack_id,
-    best_stack_id,
-    exp_id
-FROM (
-    -- Get list of warps that can be diffed, with any associated diff
-    SELECT
-        warpSkyfile.warp_id,
-        warpSkyfile.skycell_id,
-        warpSkyfile.tess_id,
-        warpSkyfile.good_frac,
-        warpRun.label,
-        filter,
-        warpRun.label as warp_label,
-        diffInputs.diff_id,
-        diffInputs.stack2 AS current_stack_id,
-        rawExp.exp_id
-    FROM warpSkyfile
-    JOIN warpRun USING(warp_id)
-    JOIN fakeRun USING(fake_id)
-    JOIN camRun USING(cam_id)
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    -- Check if it has an associated diff
-    LEFT JOIN diffInputSkyfile AS diffInputs
-        ON diffInputs.warp1 = warpSkyfile.warp_id
-        AND diffInputs.skycell_id = warpSkyfile.skycell_id
-        AND diffInputs.stack2 IS NOT NULL
-    WHERE
-        warpSkyfile.fault = 0
-        AND warpSkyfile.quality = 0
-    -- warpsToDiff WHERE hook %s
-    ) AS warpsToDiff
--- Get best stack as a function of skycell_id, filter
-JOIN (
-    SELECT
-        MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment
-        skycell_id,
-        filter
-    FROM stackRun
-    JOIN stackSumSkyfile USING(stack_id)
-    WHERE stackRun.state = 'full'
-        AND stackSumSkyfile.fault = 0
-        AND stackSumSkyfile.quality = 0
-    -- stacksForDiff WHERE hook %s
-    GROUP BY
-        skycell_id,
-        filter
-    ) AS stacksForDiff USING(skycell_id, filter)
+   exp_id,
+   warp_id,
+   rawExp.filter,
+   warpRun.label AS warpLabel,
+   warpRun.data_group AS warpDataGroup,
+   warpRun.tess_id,
+   warpSkyfile.skycell_id,
+   MAX(stack_id) AS stack_id,
+   stackRun.label AS stackLabel,
+   stackRun.data_group AS stackDataGroup,
+   diff_id,
+   diffRun.label AS diffLabel
+FROM warpRun
+     JOIN fakeRun USING(fake_id)
+     JOIN camRun USING(cam_id)
+     JOIN chipRun USING(chip_id)
+     JOIN rawExp USING(exp_id)
+     JOIN warpSkyfile USING(warp_id)
+     JOIN stackRun ON 
+       (stackRun.skycell_id = warpSkyfile.skycell_id AND
+        stackRun.filter     = rawExp.filter AND
+        stackRun.tess_id    = warpRun.tess_id)
+     JOIN stackSumSkyfile USING(stack_id)
+     LEFT JOIN diffInputSkyfile ON 
+       (warp_id = diffInputSkyfile.warp1 AND
+        stack_id = diffInputSkyfile.stack2 AND
+        diffInputSkyfile.skycell_id = stackRun.skycell_id AND
+        diffInputSkyfile.tess_id = stackRun.tess_id)
+     LEFT JOIN diffRun USING(diff_id)
+WHERE 
+     warpRun.state = 'full'
+     AND stackRun.state = 'full'
+     AND warpSkyfile.fault = 0
+     AND warpSkyfile.quality = 0
+     AND stackSumSkyfile.fault = 0
+     AND stackSumSkyfile.quality = 0
+     AND exp_id IS NOT NULL
+-- %s
+GROUP BY exp_id,warp_id,skycell_id

branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_create_tables.sql

@@ -521 +521 @@
     deteff_uq FLOAT,
     quality SMALLINT NOT NULL DEFAULT 0,
+    background_model SMALLINT,
     PRIMARY KEY(cam_id),
     KEY(fault),
@@ -1791 +1792 @@
     chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier
     chip_id BIGINT NOT NULL,          -- link to chipRun
+    cam_id BIGINT DEFAULT 0,          -- optional link to camRun
     state VARCHAR(64) NOT NULL,       -- state of run (new, full, etc.)
     workdir VARCHAR(255) NOT NULL,    -- working directory
@@ -1843 +1845 @@
     data_group VARCHAR(64),           -- group for data
     dist_group VARCHAR(64),           -- group for distribution
-    reduction VARCHAR(64),    -- reduction class (for altering recipe)
+    reduction VARCHAR(64),            -- reduction class (for altering recipe)
     note VARCHAR(255),        -- note
     registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
@@ -2122 +2124 @@
       zpt_obs FLOAT,
       zpt_stdev FLOAT,
+      fwhm_major  FLOAT,
+      fwhm_minor  FLOAT,
       quality SMALLINT NOT NULL,
       hostname VARCHAR(64) NOT NULL,
@@ -2149 +2153 @@
 
 CREATE TABLE survey (
-    surveyID    INT(8),
+    surveyID    INT,
     surveyName  VARCHAR(16) NOT NULL,
     description VARCHAR(255),
     PRIMARY KEY(surveyID),
-    UNIQUE  KEY(name)
+    UNIQUE  KEY(surveyName)
 ) ENGINE=InnoDB CHARSET=latin1;
 
-
 CREATE TABLE ippRelease (
-    rel_id      INT(8) AUTO_INCREMENT,
-    surveyID    INT(8),
-    releaseName VARCHAR(64),
-    relase_state VARCHAR(16),   -- active, pending, archive, drop 
-    dataRelease INT(8),         -- PSPS dataRelease
-    priority    INT(8),
+    rel_id      INT AUTO_INCREMENT,
+    surveyID    INT,
+    release_name VARCHAR(64),
+    release_state VARCHAR(16),   -- active, pending, archive, drop 
+    dataRelease INT,         -- PSPS dataRelease
+    priority    INT,
     PRIMARY KEY(rel_id),
-    KEY(releaseName),
-    KEY(state),
+    KEY(release_name),
+    KEY(release_state),
     KEY(priority),
     FOREIGN KEY(surveyID) REFERENCES survey(surveyID)
@@ -2173 +2176 @@
 CREATE TABLE relExp (
     relexp_id   BIGINT AUTO_INCREMENT,
-    rel_id      INT(8),
+    rel_id      INT,
     exp_id      BIGINT,
     chip_id     BIGINT,         -- links to the runs that supplied the data
     cam_id      BIGINT,         -- for this release
+    group_id    INT,         -- id of relGroup that this exposure is contained in
     state       VARCHAR(16),    -- released, pending, archive, drop  
-    flags       INT UNSIGNED,   -- flags for relphot, relastro, ??
+    flags       INT UNSIGNED,  -- flags for relphot, relastro, ??
     zpt_obs     FLOAT,          -- calibrated zero point for this release of
     zpt_stdev   FLOAT,          -- this exposure
     path_base   VARCHAR(255),   -- path_base of any supporting files for this
                                 -- release of this exposure.
+    fault SMALLINT NOT NULL,
     registered  DATETIME,       -- insertion time for this row
     time_stamp  DATETIME,       -- time of last update for this row
@@ -2188 +2193 @@
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
     KEY (state),
+    KEY (fault),
     FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id),
     FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id),
@@ -2195 +2201 @@
 CREATE TABLE relStack (
     relstack_id BIGINT AUTO_INCREMENT,
-    rel_id      INT(8),
+    rel_id      INT,
+    stack_id    BIGINT,         -- id of the stackRun
     skycal_id   BIGINT,         -- id of the sky calibration run that supplied the calibration
-                                -- for this release. We get back to stack from here
-    skycell_id  varchar(64),
-    tess_id     varchar(64),
+                                -- for this release of this skycell.
+    skycell_id  VARCHAR(64),
+    tess_id     VARCHAR(64),
+    filter      VARCHAR(16),
     state       VARCHAR(16),    -- released, pending, archive, drop  
     flags       INT UNSIGNED,
@@ -2205 +2213 @@
     zpt_obs     FLOAT,          -- calibrated zero point for this release of this skycell
     zpt_stdev   FLOAT,
-    fwhm_major  FLOAT,          -- seeing as measured by staticsky
+    mjd_obs     INT UNSIGNED,   -- for nightly stacks the mjd_obs, zero for deep and reference
     path_base   VARCHAR(255),   -- path_base of any supporting files,
+    fault SMALLINT NOT NULL,
     registered  DATETIME,
     time_stamp  DATETIME,
     PRIMARY KEY (relstack_id),
     KEY (tess_id, skycell_id),
-    KEY (rel_id, tess_id, skycell_id),
+    KEY (rel_id, tess_id, skycell_id, filter, mjd_obs),
+    KEY (stack_type),
+    KEY (mjd_obs),
+    KEY (state),
+    KEY (fault),
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
-    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
+    FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE relGroup (
+    group_id    INT AUTO_INCREMENT,
+    rel_id      INT,
+    group_type  VARCHAR(16),
+    lap_id      BIGINT,
+    group_name  VARCHAR(16),
+    state       VARCHAR(16),    -- new full
+    label       VARCHAR(64),
+    exp_list_path   VARCHAR(255),
+    fault SMALLINT NOT NULL,
+    registered  DATETIME,
+    PRIMARY KEY (group_id),
+    KEY (rel_id, group_name),
+    KEY (group_type),
+    KEY (lap_id),
+    KEY (group_name),
+    KEY (state),
+    KEY (label),
+    KEY (fault),
+    FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE lapGroup  (
+    seq_id  BIGINT,
+    projection_cell VARCHAR(64),
+    state   VARCHAR(16),
+    fault   SMALLINT,
+    PRIMARY KEY(seq_id, projection_cell),
+    KEY(state),
+    KEY(fault),
+    FOREIGN KEY(seq_id) REFERENCES lapSequence(seq_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 

branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_drop_tables.sql

@@ -109 +109 @@
 DROP TABLE IF EXISTS skycalResult;
 DROP TABLE IF EXISTS skycell;
+DROP TABLE IF EXISTS relStack;
 DROP TABLE IF EXISTS relExp;
+DROP TABLE IF EXISTS relGroup;
 DROP TABLE IF EXISTS ippRelease;
 DROP TABLE IF EXISTS survey;
+DROP TABLE IF EXISTS lapGroup;
 
 SET FOREIGN_KEY_CHECKS=1

branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelexp.sql

@@ -1 +1 @@
 SELECT
     ippRelease.rel_id,
-    ippRelease.releaseName,
+    ippRelease.release_name,
     rawExp.exp_name,
     rawExp.filter,

branches/eam_branches/ipp-20130207/ippTools/share/releasetool_listrelexp.sql

@@ -2 +2 @@
     relexp_id,
     relExp.state,
-    ippRelease.releaseName,
+    relExp.fault,
+    ippRelease.release_name,
     ippRelease.release_state,
     ippRelease.rel_id,
@@ -21 +22 @@
     camProcessedExp.path_base as cam_path_base,
     camProcessedExp.fwhm_major,
+    camProcessedExp.fwhm_minor,
     warpRun.warp_id,
     warpRun.state as warp_state,

branches/eam_branches/ipp-20130207/ippTools/share/warptool_scmap.sql

@@ -5 +5 @@
     camProcessedExp.path_base as cam_path_base,
     camProcessedExp.fault as cam_fault,
+    camProcessedExp.background_model AS cam_background_model,
     chipProcessedImfile.chip_id,
     chipRun.state,

branches/eam_branches/ipp-20130207/ippTools/src/addtool.c

@@ -988 +988 @@
     if (strcmp (stage,"cam") == 0) {
       query = pxDataGet("addtool_find_processedexp_cam.sql");
-    }
-    if (strcmp (stage,"stack") == 0) {
+    } else if (strcmp (stage,"stack") == 0) {
       query = pxDataGet("addtool_find_processedexp_stack.sql");
-    }
-    if (strcmp (stage,"staticsky") == 0) {
+    } else if (strcmp (stage,"staticsky") == 0) {
       query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql");
-    }
-    if (strcmp (stage,"skycal") == 0) {
+    } else if (strcmp (stage,"skycal") == 0) {
       query = pxDataGet("addtool_find_processedexp_skycal.sql");
+    } else {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "can't find sql file for stage %s", stage);
+        return false;
     }
 

branches/eam_branches/ipp-20130207/ippTools/src/addtoolConfig.c

@@ -125 +125 @@
     psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id",   0,            "search by add_id", 0);
     psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-stage_id",   0,            "search by stage_id", 0);
-    psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-stage",             0, "set the stage", NULL);
+    psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-stage",             0, "set the stage", "cam");
     pxcamSetSearchArgs(processedexpArgs);
     psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL);

branches/eam_branches/ipp-20130207/ippTools/src/bgtool.c

@@ -285 +285 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args,   where, "-chip_id",            "chipRun.chip_id",       "==");
+    PXOPT_COPY_S64(config->args,   where, "-cam_id",             "camRun.cam_id",         "==");
     PXOPT_COPY_S64(config->args,   where, "-exp_id",             "rawExp.exp_id",         "==");
     PXOPT_COPY_STR(config->args,   where, "-exp_name",           "rawExp.exp_name",       "==");
@@ -324 +325 @@
     PXOPT_COPY_F32(config->args,   where, "-sun_angle_min",      "rawExp.sun_angle",      ">=");
     PXOPT_COPY_F32(config->args,   where, "-sun_angle_max",      "rawExp.sun_angle",      "<");
-    pxAddLabelSearchArgs(config,   where, "-label",              "chipRun.label",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-label",              "chipRun.label",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-cam_label",          "camRun.label",          "==");
 
     if (!psListLength(where->list)) {
@@ -343 +345 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+    PXOPT_LOOKUP_STR(cam_label, config->args, "-cam_label", false, false);
+    PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", false, false);
+    if (!cam_id && !cam_label) {
+        psError(PXTOOLS_ERR_CONFIG, true, "either cam_id or cam_label is required");
+        return false;
+    }
 
     // Get chip runs to promote to chipBackgroundRun
@@ -364 +372 @@
     }
 
+    psString labelHook = psStringCopy("");
     if (!rerun) {
-        psStringAppend(&query, "\nAND chipBackgroundRun.chip_bg_id IS NULL");
         if (label) {
-            psStringAppend(&query,
-                           "\nAND (chipBackgroundRun.label = '%s'"
-                           " OR chipBackgroundRun.label IS NULL)",
-                           label);
-        }
-        if (data_group) {
-            psStringAppend(&query,
-                           "\nAND (chipBackgroundRun.data_group = '%s'"
-                           " OR chipBackgroundRun.data_group IS NULL)",
-                           data_group);
-        }
-        if (dist_group) {
-            psStringAppend(&query,
-                           "\nAND (chipBackgroundRun.dist_group = '%s'"
-                           " OR chipBackgroundRun.dist_group IS NULL)",
-                           dist_group);
-        }
+            // check for run with the newly specified label
+            psStringAppend(&labelHook, "\nAND (chipBackgroundRun.label = '%s')", label);
+        }
+        psStringAppend(&query, "\nAND chip_bg_id IS NULL");
     }
 
@@ -391 +386 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    if (!p_psDBRunQueryF(config->dbh, query, labelHook)) {
         psError(psErrorCodeLast(), false, "database error");
         psFree(query);
+        psFree(labelHook);
         if (!psDBRollback(config->dbh)) {
             psError(psErrorCodeLast(), false, "database error");
@@ -400 +396 @@
     }
     psFree(query);
+    psFree(labelHook);
 
     psArray *output = p_psDBFetchResult(config->dbh); // Matching rows
@@ -420 +417 @@
     if (pretend) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) {
             psError(psErrorCodeLast(), false, "failed to print array");
             psFree(output);
@@ -435 +432 @@
         psMetadata *md = output->data[i];
 
-        chipRunRow *row = chipRunObjectFromMetadata(md);
+        chipBackgroundRunRow *row = chipBackgroundRunObjectFromMetadata(md);
         if (!row) {
             psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun");
@@ -445 +442 @@
         }
 
-        if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, "new",
+        if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, row->cam_id, "new",
                                      workdir     ? workdir    : row->workdir,
                                      label       ? label      : row->label,
@@ -560 +557 @@
 
     if (psArrayLength(output)) {
-        if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "pendingchipBackgroundRun", !simple)) {
             psError(psErrorCodeLast(), false, "failed to print array");
             psFree(output);
@@ -621 +618 @@
         return true;
     }
-    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) {
+    if (!ippdbPrintMetadatas(stdout, output, "pendingchipBackgroundImfile", !simple)) {
         psError(psErrorCodeLast(), false, "failed to print array");
         psFree(output);
@@ -1295 +1292 @@
     PXOPT_COPY_F32(config->args,   where, "-sun_angle_max",      "rawExp.sun_angle",      "<");
     pxAddLabelSearchArgs(config,   where, "-warp_label",         "warpRun.label",         "==");
-    pxAddLabelSearchArgs(config,   where, "-chip_label",         "chipBackgroundRun.label", "==");
+    pxAddLabelSearchArgs(config,   where, "-chip_bg_label",      "chipBackgroundRun.label", "==");
+    pxAddLabelSearchArgs(config,   where, "-chip_label",         "chipRun.label",         "==");
 
     if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
@@ -1335 +1333 @@
     }
 
+    psString labelHook = psStringCopy("");
     if (!rerun) {
-        psStringAppend(&query, "\nAND warpBackgroundRun.warp_bg_id IS NULL");
         if (label) {
-            psStringAppend(&query,
-                           "\nAND (warpBackgroundRun.label = '%s'"
-                           " OR warpBackgroundRun.label IS NULL)",
-                           label);
-        }
-        if (data_group) {
-            psStringAppend(&query,
-                           "\nAND (warpBackgroundRun.data_group = '%s'"
-                           " OR warpBackgroundRun.data_group IS NULL)",
-                           data_group);
-        }
-        if (dist_group) {
-            psStringAppend(&query,
-                           "\nAND (warpBackgroundRun.dist_group = '%s'"
-                           " OR warpBackgroundRun.dist_group IS NULL)",
-                           dist_group);
-        }
+            // check for run with the newly specified label
+            psStringAppend(&labelHook, "\nAND (warpBackgroundRun.label = '%s')", label);
+        }
+        psStringAppend(&query, "\nAND warp_bg_id IS NULL");
     }
 
@@ -1362 +1347 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(psErrorCodeLast(), false, "database error");
+    if (!p_psDBRunQueryF(config->dbh, query, labelHook)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(labelHook);
         psFree(query);
         if (!psDBRollback(config->dbh)) {
@@ -1370 +1356 @@
         return false;
     }
+    psFree(labelHook);
     psFree(query);
 
@@ -1391 +1378 @@
     if (pretend) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) {
             psError(psErrorCodeLast(), false, "failed to print array");
             psFree(output);
@@ -1408 +1395 @@
         psS64 chip_bg_id = psMetadataLookupS64(NULL, md, "chip_bg_id");
 
-        warpRunRow *row = warpRunObjectFromMetadata(md);
+        
+        warpBackgroundRunRow *row = warpBackgroundRunObjectFromMetadata(md);
         if (!row) {
-            psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun");
+            psError(psErrorCodeLast(), false, "failed to convert metadata into warpRun");
             psFree(output);
             if (!psDBRollback(config->dbh)) {
@@ -1665 +1653 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_bg_id",    "warpBackgroundRun.warp_bg_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "==");
-    pxAddLabelSearchArgs(config, where, "-label",   "warpBackgroundRun.label", "LIKE");
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id",   "warpBackgroundRun.warp_bg_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-warp_id",      "warpBackgroundRun.warp_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id",   "warpBackgroundRun.chip_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",   "warpBackgroundSkyfile.skycell_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label",        "warpBackgroundRun.label", "LIKE");
     pxAddLabelSearchArgs(config, where, "-data_group",   "warpBackgroundRun.data_group", "LIKE");
     pxAddLabelSearchArgs(config, where, "-dist_group",   "warpBackgroundRun.data_group", "LIKE");

branches/eam_branches/ipp-20130207/ippTools/src/bgtoolConfig.c

@@ -50 +50 @@
     psMetadata *definechipArgs = psMetadataAlloc();
     psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0);
+    psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-cam_id", 0, "choose cam_id", 0);
     psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0);
     psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL);
@@ -89 +90 @@
     psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN);
     psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN);
-    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", 0, "search on chipRun label", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-cam_label", 0, "search on camRun label", NULL);
     psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
     psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-rerun", 0, "re-run data?", false);
@@ -279 +281 @@
     psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN);
     psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-warp_label", PS_META_DUPLICATE_OK, "search on warpRun label", NULL);
-    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_bg_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL);
     psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
     psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-rerun", 0, "rerun data?", false);
@@ -352 +355 @@
     psMetadata *warpArgs = psMetadataAlloc();
     psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0);
+    psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0);
+    psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
     psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL);
     psMetadataAddS16(warpArgs, PS_LIST_TAIL, "-fault",  0, "search by fault code", 0);

branches/eam_branches/ipp-20130207/ippTools/src/camtool.c

@@ -500 +500 @@
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
-
+    PXOPT_LOOKUP_S16(background_model, config->args, "-background_model", false, false);
+    
     PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
     PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false);
@@ -520 +521 @@
     PXOPT_LOOKUP_F32(maskfrac_max_magic, config->args, "-maskfrac_max_magic", false, false);
     PXOPT_LOOKUP_F32(maskfrac_max_advisory, config->args, "-maskfrac_max_advisory", false, false);
-
+    
     // we store actual detection efficiency by adding in zpt_obs
     PXOPT_LOOKUP_F32(deteff_inst, config->args, "-deteff_inst", false, false);
@@ -542 +543 @@
     }
 
-/*     psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", */
-/*          ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); */
     psString software_ver = NULL;
     if ((ver_pslib)&&(ver_psmodules)) {
@@ -692 +691 @@
         deteff_lq,
         deteff_uq,
-        quality
+        quality,
+        background_model
         );
 
@@ -817 +817 @@
     pxAddLabelSearchArgs (config, where, "-data_group", "camRun.data_group",     "LIKE");
     PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "==");
-
+    PXOPT_COPY_S16(config->args, where, "-background_model", "camProcessedExp.background_model", "==");
+    
     psString where2 = NULL;
     if (!pxspaceAddWhere(config, &where2, "rawExp")) {
@@ -987 +988 @@
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
     PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-cam_id",   "cam_id",   "==");
-    PXOPT_COPY_S64(config->args, where, "-chip_id",  "chip_id",  "==");
-    PXOPT_COPY_STR(config->args, where, "-class",    "class",    "==");
-    PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
-
-    if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, fault, quality)) {
+    PXOPT_LOOKUP_S16(background_model, config->args, "-set_background_model", false, false);
+    if (background_model) {
+      // CZW Copied from warptool.c
+      psMetadata *where = psMetadataAlloc();
+      psMetadata *values = psMetadataAlloc();
+      PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "==");
+      PXOPT_COPY_S16(config->args, values, "-set_background_model", "background_model", "==");
+      long rows = psDBUpdateRows(config->dbh, "camProcessedExp", where, values);
+      psFree(values);
+      psFree(where);
+      if (!rows) {
+        psError(PS_ERR_UNKNOWN, true, "no rows changed");
+      }
+    }
+    else {
+      psMetadata *where = psMetadataAlloc();
+      PXOPT_COPY_S64(config->args, where, "-cam_id",   "cam_id",   "==");
+      PXOPT_COPY_S64(config->args, where, "-chip_id",  "chip_id",  "==");
+      PXOPT_COPY_STR(config->args, where, "-class",    "class",    "==");
+      PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
+      
+      if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, fault, quality)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         psFree (where);
         return false;
-    }
-    psFree (where);
-
+      }
+      psFree (where);
+    }
     return true;
 }

branches/eam_branches/ipp-20130207/ippTools/src/camtoolConfig.c

@@ -204 +204 @@
     psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_lq", 0, "define deteff_lq", NAN);
     psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_uq", 0, "define deteff_uq", NAN);
-
+    psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-background_model", 0, "set background_model value", 0);
 
     // -processedexp
@@ -222 +222 @@
     psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0,            "only return imfiles with a fault status set", false);
+    psMetadataAddS16(processedexpArgs, PS_LIST_TAIL, "-background_model", 0, "search by background_model value", 0);
 
     // -revertprocessedexp
@@ -246 +247 @@
     psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code (required)", INT16_MAX);
     psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-set_quality",  0,            "set quality", 0);
+    psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-set_background_model", 0, "set background_model value", 0);
 
     // -block

branches/eam_branches/ipp-20130207/ippTools/src/difftool.c

@@ -1226 +1226 @@
 }
 
-
+static bool definewarpstackMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  psMetadata *where = psMetadataAlloc();
+
+  PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
+  PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE");
+  PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-warp_label", "warpRun.label", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-data_group", "warpRun.data_group", "==");
+  PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
+  PXOPT_COPY_F32(config->args, where,  "-good_frac", "warpSkyfile.good_frac", ">=");
+  PXOPT_COPY_STR(config->args, where, "-stack_label", "stackRun.label", "==");
+  PXOPT_COPY_STR(config->args, where, "-stack_data_group", "stackRun.data_group", "==");
+
+  PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
+  
+  PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option
+  PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option
+  PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); // option
+  PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+  PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+  PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+  PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false);
+  
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false);
+  PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
+  PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+  PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+
+  // Get query file
+  psString query = pxDataGet("difftool_definewarpstack.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return false;
+  }
+
+  psString whereClause = psDBGenerateWhereConditionSQL(where,NULL);
+  
+  // Don't queue things that already have diffs.
+  if (! (reRun || newTemplates) ) {
+    psStringAppend(&whereClause, "\nAND diff_id IS NULL\n");
+  }
+  // Append diff qualifiers, if we have them
+  if (label) {
+    psStringAppend(&whereClause, "\nAND ((diffRun.label = '%s') OR (diffRun.label IS NULL))",label);
+  }
+  if (data_group) {
+    psStringAppend(&whereClause, "\nAND ((diffRun.data_group = '%s') OR (diffRun.data_group IS NULL))",data_group);
+  }
+  if (reduction) {
+    psStringAppend(&whereClause, "\nAND ((diffRun.reduction = '%s') OR (diffRun.reduction IS NULL))",reduction);
+  }
+
+  psString whereClauseString = psStringCopy("");
+  psStringAppend(&whereClauseString, " \n AND %s ", whereClause);
+  //  fprintf(stderr,query,whereClauseString);
+
+  // This is just a simple query, so we don't need to do a transaction
+  if (!p_psDBRunQueryF(config->dbh, query, whereClauseString)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    psFree(whereClause);
+    return(false);
+  }
+  psFree(query);
+  psFree(whereClause);
+  psFree(whereClauseString);
+  
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+      break;
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+      break;
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+      break;
+    }
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("difftool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+  if (pretend) {
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+    psFree(output);
+    return true;
+  }
+
+  psArray *list = psArrayAllocEmpty(16); // List of runs, to print
+  long numGood = 0;
+  psS64 last_exp_id = 0;
+  psS64 diff_id = 0;
+  for (long i = 0; i < output->n; i++) {
+    psMetadata *row = output->data[i];
+    bool mdok;
+
+    psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
+    if (!mdok) {
+        psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    if (exp_id != last_exp_id) {
+      if (diff_id != 0) { // We've added a run already, and are now switching to a new one.
+        // Set state to new
+        if (!setdiffRunState(config, diff_id, "new", false)) {
+          psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, diff_id);
+          if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+          }
+          return false;
+        }
+        // Commit results
+        if (!psDBCommit(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+          psFree(list);
+          return false;
+        }
+      }
+      
+      // Begin transaction
+      if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+      }
+      // Add a new diffRun row
+      psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
+      if (!mdok) {
+        psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+      }
+      diffRunRow *run = diffRunRowAlloc(
+                                        0,        // ID
+                                        "reg",    // state
+                                        workdir,
+                                        label,
+                                        data_group ? data_group : label,
+                                        dist_group,
+                                        reduction,
+                                        NULL,     // dvodb
+                                        registered,
+                                        tess_id,
+                                        bothways, // bothways (default is false)
+                                        true,     // exposure
+                                        0,        // magicked
+                                        NULL,     // software_version
+                                        0,        // mask stat npix
+                                        NAN,      // static
+                                        NAN,      // dynamic
+                                        NAN,      // magic
+                                        NAN,      // advisory
+                                        IPP_DIFF_MODE_WARP_STACK,
+                                        note
+                                        );
+      if (!diffRunInsertObject(config->dbh, run)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(run);
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+      }
+      diff_id = psDBLastInsertID(config->dbh);
+      run->diff_id = diff_id;
+
+      psArrayAdd(list, list->n, run);
+      numGood++;
+      
+      last_exp_id = exp_id;                                     
+    } // End Adding diffRun
+    
+    if (exp_id == last_exp_id) {
+      psString skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id");
+      if (!mdok) {
+        psError(PXTOOLS_ERR_PROG, false, "skycell_id not found");
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+      }
+      psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
+      if (!mdok) {
+        psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+      }
+      psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
+      if (!mdok) {
+        psError(PXTOOLS_ERR_PROG, false, "warp_id not found");
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+      }
+      psS64 stack_id = psMetadataLookupS64(&mdok, row, "stack_id");
+      if (!mdok) {
+        psError(PXTOOLS_ERR_PROG, false, "stack_id not found");
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+      }
+
+      // Add a new skyfile row
+      diffInputSkyfileRow *skyfile = diffInputSkyfileRowAlloc(
+                                                              diff_id,   // ID
+                                                              skycell_id,
+                                                              warp_id, // warp1_id 
+                                                              PS_MAX_S64, // stack1 -> NULL
+                                                              PS_MAX_S64, // warp2_id -> NULL
+                                                              stack_id, // stack2
+                                                              tess_id,
+                                                              0 // diff_skyfile_id
+                                                              );
+      //      fprintf(stderr,"%"PRId64 " %"PRId64 " %"PRId64 " %s %s\n",diff_id,warp_id,stack_id,skycell_id, tess_id);
+      if (!diffInputSkyfileInsertObject(config->dbh, skyfile)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(list);
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+      }
+    } // End Adding skyfile
+  } // End parsing result set
+  
+  // Finish the last run's update and close the connection
+  if (diff_id != 0) {
+    if (!setdiffRunState(config, diff_id, "new", false)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, diff_id);
+      if (!psDBRollback(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+      }
+      return false;
+    }
+    // Commit results
+    if (!psDBCommit(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(list);
+      return false;
+    }
+  }
+  
+  if (numGood && !diffRunPrintObjects(stdout, list, !simple)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to print object");
+    psFree(list);
+    return(false);
+  }
+  psFree(list);
+  // Free things
+  return true;
+}  
+
+#if (0)
 static bool definewarpstackMode(pxConfig *config)
 {
@@ -1614 +1891 @@
     return true;
 }
+#endif
 
 static bool definewarpwarpMode(pxConfig *config)

branches/eam_branches/ipp-20130207/ippTools/src/disttool.c

@@ -143 +143 @@
     // optional
     PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false);
-    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false);
+    PXOPT_LOOKUP_BOOL(magic, config->args, "-magic", false);
     PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
     PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false);
@@ -175 +175 @@
             return false;
         }
-        if (no_magic) {
+        if (!magic) {
             psError(PXTOOLS_ERR_SYS, true, "no_magic forbidden with alternate inputs");
             return false;
@@ -187 +187 @@
     // an interest)
     // and exp_id to be supplied. We could add -exp_type and dateobs cuts to make it easier
-    if (!strcmp(stage, "raw") && no_magic && !(exp_id && dist_group && exp_type)) {
+    if (!strcmp(stage, "raw") && !magic && !(exp_id && dist_group && exp_type)) {
         psError(PXTOOLS_ERR_SYS, true, "exp_id, exp_type, and dist_group are required for raw stage if no_magic");
         return false;
@@ -204 +204 @@
             runJoinStr = "camRun.exp_id";
         }
-        if (!no_magic) {
+        if (magic) {
             query = pxDataGet("disttool_definebyquery_raw.sql");
         } else {
@@ -215 +215 @@
         }
 
-        if (!no_magic) {
+        if (magic) {
             if (label) {
                 psStringAppend(&query, " AND (magicDSRun.label = '%s')", label);
@@ -289 +289 @@
         }
         // fake stage doesn't require magic
-        no_magic = true;
+        magic = false;
     } else if (!strcmp(stage, "warp")) {
         magicRunType = "warpRun";
@@ -357 +357 @@
         }
         // stack stage doesn't require magic
-        no_magic = true;
+        magic = false;
     } else if (!strcmp(stage, "sky")) {
         magicRunType = "staticskyRun";
@@ -378 +378 @@
         }
         // (static)sky stage doesn't require magic
-        no_magic = true;
+        magic = false;
     } else if (!strcmp(stage, "SSdiff")) {
       magicRunType = "diffRun";
@@ -396 +396 @@
       }
 
-      no_magic = true;
+      magic = false;
     } else {
         psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage);
@@ -408 +408 @@
 
     if (!strcmp(stage, "raw")) {
-        if (!no_magic) {
+        if (magic) {
             psStringAppend(&query, " AND (magicDSRun.re_place = %d)", !use_alternate);
             psStringAppend(&query, " AND (camRun.state = 'full')");
@@ -426 +426 @@
     psString joinHook = NULL;
 
-    if (!no_magic) {
+    if (magic) {
         psStringAppend(&query, " AND (%s.magicked > 0)", magicRunType);
     }
@@ -517 +517 @@
                 NULL,     // outdir
                 clean,
-                no_magic,
+                !magic,
                 use_alternate,
                 "new",

branches/eam_branches/ipp-20130207/ippTools/src/disttoolConfig.c

@@ -48 +48 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir",   0, "define workdir (required)", NULL);
 
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed (ignored)", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-magic", 0, "magic is needed", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",    0, "define label for run", NULL);

branches/eam_branches/ipp-20130207/ippTools/src/releasetool.c

@@ -41 +41 @@
 static bool listreleaseMode(pxConfig *config);
 static bool definerelexpMode(pxConfig *config);
+static bool tocalibexpMode(pxConfig *config);
 static bool updaterelexpMode(pxConfig *config);
 static bool listrelexpMode(pxConfig *config);
+static bool definerelstackMode(pxConfig *config);
+static bool updaterelstackMode(pxConfig *config);
+static bool listrelstackMode(pxConfig *config);
+static bool definerelgroupMode(pxConfig *config);
+static bool pendingrelgroupMode(pxConfig *config);
+static bool updaterelgroupMode(pxConfig *config);
+static bool listrelgroupMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -66 +74 @@
         MODECASE(RELEASETOOL_MODE_DEFINERELEASE,    definereleaseMode);
         MODECASE(RELEASETOOL_MODE_UPDATERELEASE,    updatereleaseMode);
+        MODECASE(RELEASETOOL_MODE_TOCALIBEXP,       tocalibexpMode);
         MODECASE(RELEASETOOL_MODE_LISTRELEASE,      listreleaseMode);
 
@@ -71 +80 @@
         MODECASE(RELEASETOOL_MODE_UPDATERELEXP,     updaterelexpMode);
         MODECASE(RELEASETOOL_MODE_LISTRELEXP,       listrelexpMode);
+
+        MODECASE(RELEASETOOL_MODE_DEFINERELSTACK,   definerelstackMode);
+        MODECASE(RELEASETOOL_MODE_UPDATERELSTACK,   updaterelstackMode);
+        MODECASE(RELEASETOOL_MODE_LISTRELSTACK,     listrelstackMode);
+
+        MODECASE(RELEASETOOL_MODE_DEFINERELGROUP,   definerelgroupMode);
+        MODECASE(RELEASETOOL_MODE_UPDATERELGROUP,   updaterelgroupMode);
+        MODECASE(RELEASETOOL_MODE_PENDINGRELGROUP,  pendingrelgroupMode);
+        MODECASE(RELEASETOOL_MODE_LISTRELGROUP,     listrelgroupMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -163 +181 @@
 
     PXOPT_LOOKUP_S32(surveyID,    config->args, "-set_surveyID", true, false);
-    PXOPT_LOOKUP_STR(releaseName, config->args, "-set_releaseName", true, false);
+    PXOPT_LOOKUP_STR(release_name, config->args, "-set_release_name", true, false);
     PXOPT_LOOKUP_STR(release_state, config->args, "-set_release_state",  true, false);
     PXOPT_LOOKUP_S32(dataRelease, config->args, "-set_dataRelease", false, false);
@@ -171 +189 @@
         0,      // rel_id (auto_increment)
         surveyID,
-        releaseName,
+        release_name,
         release_state,
         dataRelease,
@@ -187 +205 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_STR(releaseName, config->args, "-releaseName", false, false);
+    PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false);
     PXOPT_LOOKUP_S32(rel_id, config->args,      "-rel_id", false, false);
 
-    if (!releaseName && !rel_id) {
-        psError(PXTOOLS_ERR_CONFIG, true, "either -releaseName or -rel_id is required\n");
-        return false;
-    }
-    PXOPT_LOOKUP_STR(release_state,       config->args, "-set_release_state",  true, false);
+    if (!release_name && !rel_id) {
+        psError(PXTOOLS_ERR_CONFIG, true, "either -release_name or -rel_id is required\n");
+        return false;
+    }
+    PXOPT_LOOKUP_STR(release_state,    config->args, "-set_release_state",  false, false);
+    PXOPT_LOOKUP_S32(priority,         config->args, "-set_priority",  false, false);
+    PXOPT_LOOKUP_S32(dataRelease,      config->args, "-set_dataRelease",  false, false);
+    if (!release_state && priority == 0 && dataRelease == -1) {
+        psError(PXTOOLS_ERR_CONFIG, true, "at least one of -set_release_state -set_priority and -set_dataRelease is required\n");
+        return false;
+    }
 
     psMetadata *where = psMetadataAlloc();
 
     PXOPT_COPY_S32(config->args, where, "-rel_id",  "rel_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-releaseName", "releaseName", "==");
-
-    psString query = NULL;
-    psStringAppend(&query, "UPDATE ippRelease SET release_state = '%s'", release_state);
+    PXOPT_COPY_STR(config->args, where, "-release_name", "release_name", "==");
+
+    psString sep = "";
+    psString comma = ", ";
+    psString query = psStringCopy("UPDATE ippRelease SET");
+    if (release_state) {
+        psStringAppend(&query, "%s release_state = '%s'", sep, release_state);
+        sep = comma;
+    }
+    if (priority != 0) {
+        psStringAppend(&query, "%s priority = %d", sep, priority);
+        sep = comma;
+    }
+    if (dataRelease != -1) {
+        psStringAppend(&query, "%s dataRelease = %d", sep, dataRelease);
+        sep = comma;
+    }
 
     psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
@@ -232 +269 @@
 
     PXOPT_COPY_STR(config->args, where, "-surveyName",  "surveyName", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-releaseName", "releaseName", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "release_name", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-release_state", "release_state", "==");
     PXOPT_COPY_S32(config->args, where, "-rel_id",  "rel_id", "==");
@@ -313 +350 @@
     // insure that at least one of these is supplied to select the release
     PXOPT_LOOKUP_S64(rel_id, config->args,      "-rel_id", false, false);
-    PXOPT_LOOKUP_STR(releaseName, config->args, "-releaseName", false, false);
-    if (!rel_id && !releaseName) {
-        psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -releaseName is required");
-        return false;
-    }
-
-    // limit query to the target release. (Note: this will select nothing of releaseName and rel_id don't match)
+    PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false);
+    if (!rel_id && !release_name) {
+        psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -release_name is required");
+        return false;
+    }
+
+    // limit query to the target release. (Note: this will select nothing if release_name and rel_id don't match)
     // note: we add these after above to insure that search args are supplied
-    PXOPT_COPY_STR(config->args, where, "-releaseName", "ippRelease.releaseName", "==");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "==");
     PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "==");
 
@@ -327 +364 @@
     PXOPT_LOOKUP_STR(state, config->args,     "-set_state", true, false);
     PXOPT_LOOKUP_U32(flags, config->args,     "-set_flags", false, false);
+    PXOPT_LOOKUP_F32(user_zpt_obs, config->args,   "-set_zpt_obs", false, false);
+    PXOPT_LOOKUP_F32(user_zpt_stdev, config->args, "-set_zpt_stdev", false, false);
     PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args,     "-set_fault", false, false);
 
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
@@ -391 +431 @@
         psS64 chip_id = psMetadataLookupS64(&status, md, "chip_id");
         psS64 cam_id = psMetadataLookupS64(&status, md, "cam_id");
-        psF32 zpt_obs = psMetadataLookupF32(&status, md, "zpt_obs");
-        psF32 zpt_stdev = psMetadataLookupF32(&status, md, "zpt_stdev");
+
+        // if zpt_obs or zpt_stdev were not supplied use the results from the camRun
+        psF32 zpt_obs, zpt_stdev;
+        if (!isfinite(user_zpt_obs)) {
+            zpt_obs = psMetadataLookupF32(&status, md, "zpt_obs");
+        } else {
+            zpt_obs = user_zpt_obs;
+        }
+        if (!isfinite(user_zpt_stdev)) {
+            zpt_stdev = psMetadataLookupF32(&status, md, "zpt_stdev");
+        } else {
+            zpt_stdev = user_zpt_stdev;
+        }
 
         if (!relExpInsert(config->dbh,
@@ -400 +451 @@
             chip_id,
             cam_id,
+            0,          // group_id
             state,
             flags,
@@ -405 +457 @@
             zpt_stdev,
             path_base,
+            fault,
             now,       // registered
             now        // time_stamp
@@ -430 +483 @@
 static bool updaterelexpMode(pxConfig *config)
 {
-    psError(PS_ERR_UNKNOWN, true, "not yet implemented");
-    return false;
-}
-
-static bool listrelexpMode(pxConfig *config)
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false);
+    PXOPT_LOOKUP_S32(rel_id, config->args,      "-rel_id", false, false);
+    PXOPT_LOOKUP_S32(relexp_id, config->args,   "-relexp_id", false, false);
+
+    if (!relexp_id && !release_name && !rel_id) {
+        psError(PXTOOLS_ERR_CONFIG, true, "at least one of -relexp_id, -release_name, and -rel_id is required\n");
+        return false;
+    }
+
+    PXOPT_LOOKUP_STR(state,    config->args, "-set_state",  false, false);
+    PXOPT_LOOKUP_U32(flags,    config->args, "-set_flags",  false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false);
+    PXOPT_LOOKUP_F32(zpt_obs,  config->args, "-set_zpt_obs",  false, false);
+    PXOPT_LOOKUP_F32(zpt_stdev,config->args, "-set_zpt_stdev",  false, false);
+    PXOPT_LOOKUP_S16(fault,    config->args, "-set_fault",  false, false);
+    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault",  false);
+    if (!state && !flags && !path_base && !isfinite(zpt_obs) && !isfinite(zpt_stdev) && !fault && !clearfault) {
+        psError(PXTOOLS_ERR_CONFIG, true, "must set at least one column\n");
+        return false;
+    }
+
+    if (fault && clearfault) {
+        psError(PXTOOLS_ERR_CONFIG, true, "cannot set and clear fault at same time\n");
+        return false;
+    }
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "==");
+    PXOPT_COPY_S32(config->args, where, "-relexp_id", "relExp.relexp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+    // make sure that we have enough parameters to identify the relExp to change
+    if (!psListLength(where->list)){
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
+        psFree(where);
+        return false;
+    }
+
+    PXOPT_COPY_S32(config->args, where, "-rel_id",  "rel_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "release_name", "==");
+
+    psString sep = "";
+    psString comma = ", ";
+    psString query = psStringCopy("UPDATE relExp JOIN ippRelease USING(rel_id) JOIN rawExp USING(exp_id) SET");
+    if (state) {
+        psStringAppend(&query, "%s relExp.state = '%s'", sep, state);
+        sep = comma;
+    }
+    if (path_base) {
+        psStringAppend(&query, "%s relExp.path_base = '%s'", sep, path_base);
+        sep = comma;
+    }
+    if (flags) {
+        psStringAppend(&query, "%s relExp.flags = %d", sep, flags);
+        sep = comma;
+    }
+    if (isfinite(zpt_obs)) {
+        psStringAppend(&query, "%s relExp.zpt_obs = %f", sep, zpt_obs);
+        sep = comma;
+    }
+    if (isfinite(zpt_stdev)) {
+        psStringAppend(&query, "%s relExp.zpt_stdev = %f", sep, zpt_stdev);
+        sep = comma;
+    }
+    if (fault) {
+        psStringAppend(&query, "%s relExp.fault = %d", sep, fault);
+        sep = comma;
+    }
+    if (clearfault) {
+        psStringAppend(&query, "%s relExp.fault = 0", sep);
+        sep = comma;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " WHERE %s", whereClause);
+    psFree(whereClause);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " ippReleases", affected);
+
+
+    psFree(query);
+
+    return true;
+}
+
+static bool tocalibexpMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
@@ -440 +583 @@
     psMetadata *where = psMetadataAlloc();
 
-    PXOPT_COPY_STR(config->args, where, "-releaseName", "ippRelease.releaseName", "LIKE");
+    PXOPT_COPY_S64(config->args, where, "-relexp_id",   "relExp.relexp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE");
+#ifdef notdef
+    PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
     pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "==");
     PXOPT_COPY_STR(config->args, where, "-state",       "relExp.state", "==");
@@ -458 +604 @@
 
     PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
+
+    PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false);
+#endif
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("releasetool_tocalibexp.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString where2 = NULL;
+#ifdef notdef
+    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxspaceAddWhere failed");
+        return false;
+    }
+#endif
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\nAND %s", whereClause);
+        psFree(whereClause);
+#ifdef notdef
+    } else if (where2) {
+        psStringAppend(&query, "\nAND ");
+#endif
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
+        psFree(where);
+        return false;
+    }
+
+    if (where2) {
+        psStringAppend(&query, "\n%s", where2);
+        psFree(where2);
+    }
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!psArrayLength(output)) {
+        psTrace("releasetool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+static bool listrelexpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-relexp_id",   "relExp.relexp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-state",       "relExp.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter",      "rawExp.filter", "LIKE");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "camProcessedExp.fwhm_major", ">=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "camProcessedExp.fwhm_major", "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_name",    "rawExp.exp_name", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id",      "relExp.exp_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",     "relExp.chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",      "relExp.cam_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-warp_id",     "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-cam_data_group",  "camRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE");
+
+    PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
     PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
 
@@ -533 +779 @@
     return true;
 }
-
+static bool definerelstackMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+
+    bool includeSkycal = false;
+
+    PXOPT_COPY_S64(config->args, where, "-skycal_id",  "skycalRun.skycal_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "==");
+    if (psListLength(where->list)) {
+        includeSkycal = true;
+    }
+
+    PXOPT_COPY_STR(config->args, where, "-label",      "stackRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "==");
+    PXOPT_COPY_S64(config->args, where, "-stack_id",   "stackRun.stack_id", "==");
+
+    // Insure that at least one of the skycalRun selectors is supplied
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "at least one of -label, -data_group, or -skycal_id is required");
+        return false;
+    }
+
+    // PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackRun.tess_id", "==");
+    // PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
+
+    // insure that at least one of these is supplied to select the release
+    PXOPT_LOOKUP_S64(rel_id, config->args,      "-rel_id", false, false);
+    PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false);
+    if (!rel_id && !release_name) {
+        psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -release_name is required");
+        return false;
+    }
+
+    // limit query to the target release. (Note: this will select nothing if release_name and rel_id don't match)
+    // note: we add these after above to insure that search args are supplied
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "==");
+    PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "==");
+
+
+    PXOPT_LOOKUP_STR(state, config->args,     "-set_state", true, false);
+    PXOPT_LOOKUP_STR(stack_type, config->args,"-set_stack_type", true, false);
+    PXOPT_LOOKUP_U32(flags, config->args,     "-set_flags", false, false);
+    PXOPT_LOOKUP_F32(zpt_obs, config->args,   "-set_zpt_obs", false, false);
+    PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-set_zpt_stdev", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args,     "-set_fault", false, false);
+
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+    PXOPT_LOOKUP_BOOL(simple, config->args,  "-simple", false);
+
+    // find the parameters of all the exposures that we want to add to the release
+    psString query;
+    
+    if (includeSkycal) {
+        query = pxDataGet("releasetool_definerelstack_with_skycal.sql");
+    } else {
+        query = pxDataGet("releasetool_definerelstack.sql");
+    }
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("releasetool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "relstack", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(output);
+        return false;
+    }
+
+    psTime *now = psTimeGetNow(PS_TIME_TAI);
+    // loop over our list of  exposures
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        bool status = true;
+        psS32 rel_id = psMetadataLookupS32(&status, md, "rel_id");
+        psS64 stack_id = psMetadataLookupS64(&status, md, "stack_id");
+        psS64 skycal_id = psMetadataLookupS64(&status, md, "skycal_id");
+        psString skycell_id = psMetadataLookupStr(&status, md, "skycell_id");
+        psString tess_id = psMetadataLookupStr(&status, md, "tess_id");
+        psString filter = psMetadataLookupStr(&status, md, "filter");
+
+        // use skycal zero point measurement if not supplied with arguments
+        psF32 sc_zpt_obs = psMetadataLookupF32(&status, md, "zpt_obs");
+        psF32 sc_zpt_stdev = psMetadataLookupF32(&status, md, "zpt_stdev");
+        psF32 stack_mjd_obs = psMetadataLookupF32(&status, md, "mjd_obs");
+
+        psU32 mjd_obs = 0;
+        if (!strcmp(stack_type, "nightly")) {
+            // only for nightly stacks do we set mjd_obs to a non zero value
+            // We use integer day because this is part of the key that is used
+            // to make sure entries are unique
+            //      KEY (rel_id, tess_id, skycell_id, filter, mjd_obs),
+            // 1 entry per release, skycell, filter and mjd_obs. 
+            // mjd_obs is zero for deep and reference stacks
+            mjd_obs = (psU32) stack_mjd_obs;
+        }
+
+        if (!relStackInsert(config->dbh,
+            0,          // relstack_id (auto increment)
+            rel_id,
+            stack_id,
+            skycal_id,
+            skycell_id,
+            tess_id,
+            filter,
+            state,
+            flags,
+            stack_type,
+            isfinite(zpt_obs)   ? zpt_obs   : sc_zpt_obs,
+            isfinite(zpt_stdev) ? zpt_stdev : sc_zpt_stdev,
+            mjd_obs,
+            path_base,
+            fault,
+            now,       // registered
+            now        // time_stamp
+        )) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "failed to trying to insert relStack for stack_id %" PRId64, stack_id);
+            psFree(output);
+            psFree(now);
+            return false;
+        }
+    }
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool updaterelstackMode(pxConfig *config)
+{
+    psError(PS_ERR_UNKNOWN, true, "not yet implemented");
+    return false;
+}
+
+static bool listrelstackMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-state",       "relExp.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter",      "rawExp.filter", "LIKE");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_major", ">=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_name",    "rawExp.exp_ame", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id",      "relExp.exp_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",     "relExp.chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",      "relExp.cam_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-warp_id",     "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-cam_data_group",  "camRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE");
+
+    PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
+    PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
+
+    PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("releasetool_listrelexp.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString where2 = NULL;
+    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxspaceAddWhere failed");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    } else if (where2) {
+        psStringAppend(&query, "\nWHERE ");
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
+        psFree(where);
+        return false;
+    }
+
+    if (where2) {
+        psStringAppend(&query, "\n%s", where2);
+        psFree(where2);
+    }
+
+    if (priority_order) {
+        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority");
+    }
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!psArrayLength(output)) {
+        psTrace("releasetool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool definerelgroupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_STR(group_type, config->args, "-set_group_type", true, false);
+    PXOPT_LOOKUP_STR(group_name, config->args, "-set_group_name", false, false);
+    PXOPT_LOOKUP_STR(label, config->args,     "-set_label", true, false);
+
+    PXOPT_LOOKUP_STR(state, config->args,     "-set_state", false, false);
+    PXOPT_LOOKUP_STR(exp_list_path, config->args, "-set_exp_list_path", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args,     "-set_fault", false, false);
+
+    psString query = NULL;
+
+    psMetadata *where = psMetadataAlloc();
+
+    // select sql and paramters based on group type
+    bool type_lap = false;
+    if (!strcmp(group_type, "lap")) {
+        type_lap = true;
+        if (group_name) {
+            psError(PXTOOLS_ERR_CONFIG, true, "group_name is not allowed with group_type lap");
+            psFree(where);
+            return false;
+        }
+        PXOPT_COPY_S64(config->args, where, "-select_seq_id", "lapRun.seq_id", "==");
+        PXOPT_COPY_S64(config->args, where, "-select_lap_id", "lapRun.lap_id", "==");
+        query = pxDataGet("releasetool_definerelgroup_select_lap.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+    } else {
+        if (!strcmp(group_type, "date")) {
+            // group_type ok
+        } else if (!strcmp(group_type, "data_group")) {
+            // group_type ok
+        } else {
+            psError(PXTOOLS_ERR_CONFIG, true, "%s is not valid group_type", group_type);
+            return false;
+        }
+        if (!group_name) {
+            psError(PXTOOLS_ERR_CONFIG, true, "group_name is required allowed with group_type %s", group_type);
+            psFree(where);
+            return false;
+        }
+        pxAddLabelSearchArgs (config, where, "-select_data_group", "camRun.data_group", "==");
+        query = pxDataGet("releasetool_definerelgroup_select_data_group.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+    }
+
+    // Insure that at least one of the exposure selectors is supplied
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search arguments are required");
+        return false;
+    }
+
+    // insure that at least one of these is supplied to select the release
+    PXOPT_LOOKUP_S64(rel_id, config->args,      "-rel_id", false, false);
+    PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false);
+    if (!rel_id && !release_name) {
+        psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -release_name is required");
+        return false;
+    }
+
+    // limit query to the target release. (Note: this will select nothing if release_name and rel_id don't match)
+    // note: we add these after above to insure that search args are supplied
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "==");
+    PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "==");
+
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+    PXOPT_LOOKUP_BOOL(simple, config->args,  "-simple", false);
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+
+    psFree(where);
+
+    if (!type_lap) {
+        psStringAppend(&query, "\nHAVING COUNT(exp_id) > 0");
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("releasetool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "relGroupToQueue", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(output);
+        return false;
+    }
+
+    psTime *now = psTimeGetNow(PS_TIME_TAI);
+    // loop over our list of  exposures
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        bool status = true;
+        psS32 rel_id = psMetadataLookupS32(&status, md, "rel_id");
+        psS64 lap_id = 0;
+        if (type_lap) {
+            lap_id = psMetadataLookupS64(&status, md, "lap_id");
+        }
+
+        if (!relGroupInsert(config->dbh,
+            0,          // group_id (auto increment)
+            rel_id,
+            group_type,
+            lap_id,
+            group_name,
+            "reg",      // state
+            label,
+            exp_list_path,
+            fault,
+            now        // registered
+        )) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "failed to trying to insert relGroup");
+            psFree(output);
+            psFree(now);
+            return false;
+        }
+        // Here's our new group_id
+        psS32 group_id = psDBLastInsertID(config->dbh);
+
+        psString query;
+        if (type_lap) {
+            query = pxDataGet("releasetool_definerelgroup_select_exp_lap.sql");
+            if (!query) {
+                psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+                psFree(where);
+                return false;
+            }
+            psStringAppend(&query, " AND relExp.rel_id = %d AND lap_id = %" PRId64, rel_id, lap_id);
+        } else {
+            query = pxDataGet("releasetool_definerelgroup_select_exp_data_group.sql");
+            if (!query) {
+                psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+                psFree(where);
+                return false;
+            }
+            psStringAppend(&query, " AND relExp.rel_id = %d", rel_id);
+
+            psMetadata *where = psMetadataAlloc();
+            pxAddLabelSearchArgs (config, where, "-select_data_group", "camRun.data_group", "==");
+            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringAppend(&query, " AND %s", whereClause);
+            psFree(whereClause);
+            psFree(where);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(query);
+            psFree(output);
+            psFree(now);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(query);
+
+        psArray *exposures = p_psDBFetchResult(config->dbh);
+        if (!exposures) {
+            psFree(now);
+            psFree(output);
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        if (!psArrayLength(exposures)) {
+            psFree(now);
+            psFree(output);
+            psFree(exposures);
+            psError(PS_ERR_UNKNOWN, false, "no exposures found error");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        // assign the relExps to this new group
+        char *updateQuery = "UPDATE relExp SET group_id = %d WHERE relexp_id = %" PRId64;
+        for (int j = 0; j < psArrayLength(exposures); j++) {
+            psMetadata *md = exposures->data[j];
+
+            bool status = true;
+            psS32 relexp_id = psMetadataLookupS32(&status, md, "relexp_id");
+            if (!p_psDBRunQueryF(config->dbh, updateQuery, group_id, relexp_id)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(now);
+                psFree(output);
+                psFree(exposures);
+
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                return false;
+            }
+        }
+        psFree(exposures);
+
+        // set state of relGroup to 'new'
+        if (!p_psDBRunQueryF(config->dbh, "UPDATE relGroup set state = 'new' WHERE group_id = %d", group_id)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(now);
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+    }
+    psFree(output);
+    psFree(now);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool updaterelgroupMode(pxConfig *config)
+{
+    psError(PS_ERR_UNKNOWN, true, "not yet implemented");
+    return false;
+}
+
+static bool pendingrelgroupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+
+    pxAddLabelSearchArgs (config, where, "-label",      "relGroup.label", "==");
+    PXOPT_COPY_S32(config->args, where, "-group_id",    "relGroup.group_id", "==");
+    PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("releasetool_pendingrelgroup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\nAND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
+        psFree(where);
+        return false;
+    }
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!psArrayLength(output)) {
+        psTrace("releasetool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (!ippdbPrintMetadatas(stdout, output, "pending_relGroup", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool listrelgroupMode(pxConfig *config)
+{
+    return false;
+#ifdef notdef
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-state",       "relExp.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter",      "rawExp.filter", "LIKE");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_major", ">=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_name",    "rawExp.exp_ame", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id",      "relExp.exp_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",     "relExp.chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",      "relExp.cam_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-warp_id",     "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-cam_data_group",  "camRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE");
+
+    PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
+    PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
+
+    PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("releasetool_listrelexp.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString where2 = NULL;
+    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxspaceAddWhere failed");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    } else if (where2) {
+        psStringAppend(&query, "\nWHERE ");
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n");
+        psFree(where);
+        return false;
+    }
+
+    if (where2) {
+        psStringAppend(&query, "\n%s", where2);
+        psFree(where2);
+    }
+
+    if (priority_order) {
+        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority");
+    }
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!psArrayLength(output)) {
+        psTrace("releasetool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+#endif //notdef
+}
+

branches/eam_branches/ipp-20130207/ippTools/src/releasetool.h

@@ -32 +32 @@
     RELEASETOOL_MODE_DEFINERELEXP,
     RELEASETOOL_MODE_UPDATERELEXP,
+    RELEASETOOL_MODE_TOCALIBEXP,
     RELEASETOOL_MODE_LISTRELEXP,
+    RELEASETOOL_MODE_DEFINERELSTACK,
+    RELEASETOOL_MODE_UPDATERELSTACK,
+    RELEASETOOL_MODE_TOCALIBSTACK,
+    RELEASETOOL_MODE_LISTRELSTACK,
+    RELEASETOOL_MODE_DEFINERELGROUP,
+    RELEASETOOL_MODE_PENDINGRELGROUP,
+    RELEASETOOL_MODE_UPDATERELGROUP,
+    RELEASETOOL_MODE_LISTRELGROUP,
 } releasetoolMode;
 

branches/eam_branches/ipp-20130207/ippTools/src/releasetoolConfig.c

@@ -60 +60 @@
     psMetadata *definereleaseArgs = psMetadataAlloc();
     psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_surveyID", 0,      "define survey ID (required)", 0);
-    psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_releaseName", 0,   "define release name (required)", NULL);
+    psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_release_name", 0,   "define release name (required)", NULL);
     psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0,   "define dataRelease", -1);
     psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0, "define release state", NULL);
-    psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_priority", 0,      "define release priority", -1);
+    psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_priority", 0,      "define release priority", 0);
 
     // -uptdaterelease
     psMetadata *updatereleaseArgs = psMetadataAlloc();
 
-    // releaseName and/or rel_id is required. This is handled in updatereleaseMode
-    psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-releaseName", 0,        "select by release name", NULL);
+    // release_name and/or rel_id is required. This is handled in updatereleaseMode
+    psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-release_name", 0,        "select by release name", NULL);
     psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-rel_id", 0,             "select by release ID", 0);
 
-    // Only support changing state and priority for now
-    psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0,  "define new release state(required)", NULL);
+    psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0,  "define new release state", NULL);
     psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_priority", 0,       "define new release priority", 0);
-
-//    psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_surveyID", 0,       "define surveyID (required)", 0);
-//    psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_releaseName", 0,    "define release name (required)", NULL);
-//    psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0,    "define dataRelease", -1);
+    psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0,    "define data release", -1);
 
 
@@ -84 +80 @@
     psMetadata *listreleaseArgs = psMetadataAlloc();
     psMetadataAddStr(listreleaseArgs,  PS_LIST_TAIL, "-surveyName", 0,  "select by survey name (LIKE comparision)", NULL);
-    psMetadataAddStr(listreleaseArgs,  PS_LIST_TAIL, "-releaseName", 0, "select by release name (LIKE comparision)", NULL);
+    psMetadataAddStr(listreleaseArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
     psMetadataAddStr(listreleaseArgs,  PS_LIST_TAIL, "-release_state",  0, "select by release state", NULL);
     psMetadataAddS32(listreleaseArgs, PS_LIST_TAIL,  "-rel_id", 0,      "select by release ID", 0);
@@ -95 +91 @@
 
         // set the target release
-    psMetadataAddStr(definerelexpArgs,  PS_LIST_TAIL, "-releaseName", 0, "define release name", NULL);
+    psMetadataAddStr(definerelexpArgs,  PS_LIST_TAIL, "-release_name", 0, "define release name", NULL);
     psMetadataAddS32(definerelexpArgs, PS_LIST_TAIL,  "-rel_id", 0,      "define release ID", 0);
 
@@ -109 +105 @@
         // parameters of the relExp
     psMetadataAddStr(definerelexpArgs,  PS_LIST_TAIL, "-set_state", 0,      "define state (required)", NULL);
+    psMetadataAddU32(definerelexpArgs,  PS_LIST_TAIL, "-set_flags", 0,      "define flags", 0);
+    psMetadataAddF32(definerelexpArgs,  PS_LIST_TAIL, "-set_zpt_obs", 0,    "define zero point", NAN);
+    psMetadataAddF32(definerelexpArgs,  PS_LIST_TAIL, "-set_zpt_stdev", 0,  "define zero point stdev", NAN);
     psMetadataAddStr(definerelexpArgs,  PS_LIST_TAIL, "-set_path_base", 0,  "define state", NULL);
-    psMetadataAddU32(definerelexpArgs,  PS_LIST_TAIL, "-set_flags", 0,      "define flags", 0);
-
-    psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-pretend",  0, "do not actual modify the database", false);
-    psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-simple",  0, "use the simpel output format", false);
-
-
+    psMetadataAddS16(definerelexpArgs,  PS_LIST_TAIL, "-set_fault", 0,      "define fault", 0);
+
+    psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-pretend",  0, "do not actually modify the database", false);
+    psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
 
     // -updaterelexp
     psMetadata *updaterelexpArgs = psMetadataAlloc();
-    psMetadataAddStr(updaterelexpArgs,  PS_LIST_TAIL, "-releaseName", 0, "select by release name (LIKE comparision)", NULL);
+    psMetadataAddS64(updaterelexpArgs, PS_LIST_TAIL,  "-relexp_id", 0,   "select by released exposure ID", 0);
+    psMetadataAddS64(updaterelexpArgs, PS_LIST_TAIL,  "-exp_id", 0,      "select by exposure ID", 0);
+    psMetadataAddStr(updaterelexpArgs,  PS_LIST_TAIL, "-exp_name", 0,    "select by exposure name", NULL);
+    psMetadataAddS32(updaterelexpArgs, PS_LIST_TAIL,  "-rel_id", 0,      "select by release ID", 0);
+    psMetadataAddStr(updaterelexpArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
+
+    psMetadataAddStr(updaterelexpArgs,  PS_LIST_TAIL, "-set_state", 0,      "define new state)", NULL);
+    psMetadataAddU32(updaterelexpArgs,  PS_LIST_TAIL, "-set_flags", 0,      "define new flags", 0);
+    psMetadataAddF32(updaterelexpArgs,  PS_LIST_TAIL, "-set_zpt_obs", 0,    "define new zero point", NAN);
+    psMetadataAddF32(updaterelexpArgs,  PS_LIST_TAIL, "-set_zpt_stdev", 0,  "define new zero point stdev", NAN);
+    psMetadataAddStr(updaterelexpArgs,  PS_LIST_TAIL, "-set_path_base", 0,  "define path_base", NULL);
+    psMetadataAddS16(updaterelexpArgs,  PS_LIST_TAIL, "-set_fault", 0,      "define new fault", 0);
+    psMetadataAddBool(updaterelexpArgs, PS_LIST_TAIL, "-clearfault",  0,   "set fault to zero", false);
+
+    // -tocalibexp
+    psMetadata *tocalibexpArgs = psMetadataAlloc();
+    psMetadataAddS64(tocalibexpArgs, PS_LIST_TAIL,  "-relexp_id", 0,   "select by released exposure ID", 0);
+    psMetadataAddStr(tocalibexpArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
+
+    psMetadataAddU64(tocalibexpArgs,  PS_LIST_TAIL, "-limit",       0, "limit result set to N items", 0);
+    psMetadataAddBool(tocalibexpArgs, PS_LIST_TAIL, "-simple",      0, "use the simple output format", false);
 
     // -listrelexp
     psMetadata *listrelexpArgs = psMetadataAlloc();
-    psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-releaseName", 0,  "select by release name (LIKE comparision)", NULL);
+    psMetadataAddS64(listrelexpArgs, PS_LIST_TAIL,  "-relexp_id", 0,   "select by released exposure ID", 0);
+    psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-release_state", PS_META_DUPLICATE_OK, "select by release state", NULL);
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-state", 0,        "select by released exposure state", NULL);
@@ -155 +173 @@
     psMetadataAddBool(listrelexpArgs, PS_LIST_TAIL, "-simple",      0, "use the simple output format", false);
 
+    // -definerelstack
+    psMetadata *definerelstackArgs = psMetadataAlloc();
+
+        // set the target release
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-release_name", 0, "define release name", NULL);
+    psMetadataAddS32(definerelstackArgs, PS_LIST_TAIL,  "-rel_id", 0,      "define release ID", 0);
+
+        // select the processing
+    psMetadataAddS64(definerelstackArgs, PS_LIST_TAIL,  "-stack_id", 0,    "select by stack ID", 0);
+    psMetadataAddS64(definerelstackArgs, PS_LIST_TAIL,  "-skycal_id", 0,   "select by skycal ID", 0);
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-label",  0,      "select by stackRun.label", NULL);
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-data_group", 0,  "select by stackRun.data_group", NULL);
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-skycal_label",  0, "select by skycalRun.label", NULL);
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-skycal_data_group", 0,  "select by skycalRun.data_group", NULL);
+
+        // parameters of the relStack
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-set_state", 0,      "define state (required)", NULL);
+    psMetadataAddU32(definerelstackArgs,  PS_LIST_TAIL, "-set_flags", 0,      "define flags", 0);
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-set_stack_type", 0, "define stack type (required)", NULL);
+    psMetadataAddF32(definerelstackArgs,  PS_LIST_TAIL, "-set_zpt_obs", 0,    "define zero point", NAN);
+    psMetadataAddF32(definerelstackArgs,  PS_LIST_TAIL, "-set_zpt_stdev", 0,  "define zero point stdev", NAN);
+    psMetadataAddF32(definerelstackArgs,  PS_LIST_TAIL, "-set_fwhm_major", 0, "define fwhm_major", NAN);
+    psMetadataAddStr(definerelstackArgs,  PS_LIST_TAIL, "-set_path_base", 0,  "define state (required)", NULL);
+    psMetadataAddS16(definerelstackArgs,  PS_LIST_TAIL, "-set_fault", 0,      "define fault", 0);
+
+    psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-pretend",  0, "do not actually modify the database", false);
+    psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+
+    // -definerelgroup
+    psMetadata *definerelgroupArgs = psMetadataAlloc();
+
+        // set the target release
+    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-release_name", 0, "define release name", NULL);
+    psMetadataAddS32(definerelgroupArgs, PS_LIST_TAIL, "-rel_id", 0,      "define release ID", 0);
+
+        // select the members of the group
+    psMetadataAddS64(definerelgroupArgs, PS_LIST_TAIL, "-select_seq_id", 0,  "select by LAP sequence ID", 0);
+    psMetadataAddS64(definerelgroupArgs, PS_LIST_TAIL, "-select_lap_id", 0,  "select by LAP ID", 0);
+    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK,  "select by camRun.data_group", NULL);
+    // do we want this?
+    // psMetadataAddS64(definerelgroupArgs, PS_LIST_TAIL, "-lap_id", 0,  "select by LAP ID", NULL);
+
+        // parameters of the relGroup
+    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_type", 0,  "define group_type (required)", NULL);
+    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0,  "define group_name (required)", NULL);
+    psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_label", 0, "define relgroup label (required)", NULL);
+    psMetadataAddStr(definerelgroupArgs,  PS_LIST_TAIL, "-set_state", 0,      "define state", NULL);
+    psMetadataAddStr(definerelgroupArgs,  PS_LIST_TAIL, "-set_exp_list_path", 0, "define path in exposure list", NULL);
+    psMetadataAddS16(definerelgroupArgs,  PS_LIST_TAIL, "-set_fault", 0,      "define fault", 0);
+
+        // options
+    psMetadataAddBool(definerelgroupArgs, PS_LIST_TAIL, "-pretend",  0, "do not actually modify the database", false);
+    psMetadataAddBool(definerelgroupArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+
+    // -pendingrelgroup
+    psMetadata *pendingrelgroupArgs = psMetadataAlloc();
+
+    psMetadataAddStr(pendingrelgroupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK,  "search by relGroup.label", NULL);
+    psMetadataAddS32(pendingrelgroupArgs, PS_LIST_TAIL, "-rel_id", 0,      "select by release ID", 0);
+    psMetadataAddS32(pendingrelgroupArgs, PS_LIST_TAIL, "-group_id", 0,    "select by relGroup ID", 0);
+    psMetadataAddStr(pendingrelgroupArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name", 0);
+    psMetadataAddBool(pendingrelgroupArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+    psMetadataAddU64(pendingrelgroupArgs, PS_LIST_TAIL,  "-limit",       0, "limit result set to N items", 0);
+    
+    // -updaterelgroup
+    psMetadata *updaterelgroupArgs = psMetadataAlloc();
+
+        // set the target release
+    psMetadataAddS32(updaterelgroupArgs,  PS_LIST_TAIL, "-group_id", 0,   "select by group ID (required)", 0);
+
+        // parameters of the relGroup
+    psMetadataAddStr(updaterelgroupArgs,  PS_LIST_TAIL, "-set_state", 0,  "define state", NULL);
+    psMetadataAddStr(updaterelgroupArgs,  PS_LIST_TAIL, "-set_exp_list_path", 0, "define path in exposure list", NULL);
+    psMetadataAddStr(updaterelgroupArgs,  PS_LIST_TAIL, "-set_label", 0,  "define relgroup label", NULL);
+    psMetadataAddS16(updaterelgroupArgs,  PS_LIST_TAIL, "-set_fault", 0,  "define fault", 0);
+    psMetadataAddBool(updaterelgroupArgs, PS_LIST_TAIL, "-clearfault",  0, "set fault to zero", false);
+
+    psMetadata *listrelgroupArgs = psMetadataAlloc();
+    psMetadataAddS32(listrelgroupArgs,  PS_LIST_TAIL, "-group_id", 0,   "select by group ID", 0);
+    psMetadataAddS32(listrelgroupArgs,  PS_LIST_TAIL, "-rel_id", 0,   "select by release ID", 0);
+    psMetadataAddStr(listrelgroupArgs, PS_LIST_TAIL,  "-release_name", 0, "select by release name", 0);
+
+    // ******************************************************************
+
     psMetadata *argSets = psMetadataAlloc();
     psMetadata *modes = psMetadataAlloc();
@@ -160 +262 @@
     PXOPT_ADD_MODE("-definesurvey",     "define a survey",              RELEASETOOL_MODE_DEFINESURVEY,  definesurveyArgs);
     PXOPT_ADD_MODE("-listsurvey",       "list skycell parameters",      RELEASETOOL_MODE_LISTSURVEY,    listsurveyArgs);
+
     PXOPT_ADD_MODE("-definerelease",    "define a release",             RELEASETOOL_MODE_DEFINERELEASE, definereleaseArgs);
     PXOPT_ADD_MODE("-updaterelease",    "update a release",             RELEASETOOL_MODE_UPDATERELEASE, updatereleaseArgs);
     PXOPT_ADD_MODE("-listrelease",      "list releases",                RELEASETOOL_MODE_LISTRELEASE,   listreleaseArgs);
+
     PXOPT_ADD_MODE("-definerelexp",     "define a released exposure",   RELEASETOOL_MODE_DEFINERELEXP,  definerelexpArgs);
     PXOPT_ADD_MODE("-updaterelexp",     "update a released exposure",   RELEASETOOL_MODE_UPDATERELEXP,  updaterelexpArgs);
+    PXOPT_ADD_MODE("-tocalibexp",       "list relExps pending calibration",  RELEASETOOL_MODE_TOCALIBEXP,  tocalibexpArgs);
     PXOPT_ADD_MODE("-listrelexp",       "list released exposures",      RELEASETOOL_MODE_LISTRELEXP,    listrelexpArgs);
+
+    PXOPT_ADD_MODE("-definerelstack",     "define a released stack",    RELEASETOOL_MODE_DEFINERELSTACK,  definerelstackArgs);
+
+    PXOPT_ADD_MODE("-definerelgroup",     "define a group of exposures", RELEASETOOL_MODE_DEFINERELGROUP,  definerelgroupArgs);
+    PXOPT_ADD_MODE("-pendingrelgroup",     "list relGroups pending processing",    RELEASETOOL_MODE_PENDINGRELGROUP,  pendingrelgroupArgs);
+    PXOPT_ADD_MODE("-updaterelgroup",     "update a relGroup",          RELEASETOOL_MODE_UPDATERELGROUP,  updaterelgroupArgs);
+    PXOPT_ADD_MODE("-listrelgroup",     "define a group of exposures", RELEASETOOL_MODE_LISTRELGROUP,  listrelgroupArgs);
+
+
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/eam_branches/ipp-20130207/ippTools/src/staticskytool.c

@@ -1281 +1281 @@
     PXOPT_LOOKUP_F32(zpt_obs, config->args,   "-zpt_obs", false, false);
     PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-zpt_stdev", false, false);
+    PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false);
+    PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false);
 
 
@@ -1334 +1336 @@
         zpt_obs,
         zpt_stdev,
+        fwhm_major,
+        fwhm_minor,
         quality,
         software_ver,

branches/eam_branches/ipp-20130207/ippTools/src/staticskytoolConfig.c

@@ -211 +211 @@
     psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_ra", 0,            "define exposure E ra", NAN);
     psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_dec", 0,            "define exposure E dec", NAN);
-    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0,   "define observed zero point", 0);
-    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0,   "define observed zero point stdandard deviation", 0);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0,   "define observed zero point", NAN);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0,   "define observed zero point stdandard deviation", NAN);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-fwhm_major", 0,   "define fwhm (major axis; pixels)", NAN);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-fwhm_minor", 0,   "define fwhm (minor axis; pixels)", NAN);
 
     psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN);

branches/eam_branches/ipp-20130207/ippconfig/dvo.photcodes

@@ -20 +20 @@
 # PS1 ISP:  g: 3200,      r: 3300,      i: 3400,      z: 3500,      y: 3600
 # PS1 TC3:  g: 3280-3296, r: 3380-3396, i: 3480-3496, z: 3580-3596, y: 3680-3696, 
-# PS1 GPC1: g: 3201-3264, r: 3301-3364, i: 3401-3464, z: 3501-3564, y: 3601-3664
+# PS1 GPC1: g: 3201-3264, r: 3301-3364, i: 3401-3464, z: 3501-3564, y: 3601-3664, w: 3201-3464
 
 # NOTE 2012.03.30 EAM : I have added 'poor' and 'bad' bitsets for GPC1 data using psphot
@@ -37 +37 @@
   7     H                    sec   0.000  0.000 0.000     -     - 0.0000     0  2012   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
   8     K                    sec   0.000  0.000 0.000     -     - 0.0000     0  2013   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
+  9     w                    sec   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
 
   1010  U_L92                ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
@@ -835 +836 @@
   10475 GPC1.y.XY75          dep  23.320 -0.073 0.000     -     - 0.0000     0     5   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
   10476 GPC1.y.XY76          dep  23.320 -0.073 0.000     -     - 0.0000     0     5   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10501 GPC1.w.XY01          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10502 GPC1.w.XY02          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10503 GPC1.w.XY03          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10504 GPC1.w.XY04          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10505 GPC1.w.XY05          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10506 GPC1.w.XY06          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10510 GPC1.w.XY10          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10511 GPC1.w.XY11          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10512 GPC1.w.XY12          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10513 GPC1.w.XY13          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10514 GPC1.w.XY14          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10515 GPC1.w.XY15          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10516 GPC1.w.XY16          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10517 GPC1.w.XY17          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10520 GPC1.w.XY20          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10521 GPC1.w.XY21          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10522 GPC1.w.XY22          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10523 GPC1.w.XY23          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10524 GPC1.w.XY24          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10525 GPC1.w.XY25          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10526 GPC1.w.XY26          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10527 GPC1.w.XY27          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10530 GPC1.w.XY30          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10531 GPC1.w.XY31          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10532 GPC1.w.XY32          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10533 GPC1.w.XY33          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10534 GPC1.w.XY34          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10535 GPC1.w.XY35          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10536 GPC1.w.XY36          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10537 GPC1.w.XY37          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10540 GPC1.w.XY40          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10541 GPC1.w.XY41          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10542 GPC1.w.XY42          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10543 GPC1.w.XY43          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10544 GPC1.w.XY44          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10545 GPC1.w.XY45          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10546 GPC1.w.XY46          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10547 GPC1.w.XY47          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10550 GPC1.w.XY50          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10551 GPC1.w.XY51          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10552 GPC1.w.XY52          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10553 GPC1.w.XY53          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10554 GPC1.w.XY54          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10555 GPC1.w.XY55          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10556 GPC1.w.XY56          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10557 GPC1.w.XY57          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10560 GPC1.w.XY60          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10561 GPC1.w.XY61          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10562 GPC1.w.XY62          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10563 GPC1.w.XY63          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10564 GPC1.w.XY64          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10565 GPC1.w.XY65          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10566 GPC1.w.XY66          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10567 GPC1.w.XY67          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10571 GPC1.w.XY71          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10572 GPC1.w.XY72          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10573 GPC1.w.XY73          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10574 GPC1.w.XY74          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10575 GPC1.w.XY75          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
+  10576 GPC1.w.XY76          dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88
   11000 GPC1.g.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
   11100 GPC1.r.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
@@ -840 +901 @@
   11300 GPC1.z.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
   11400 GPC1.y.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
+  11500 GPC1.w.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     9   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
   160   MOSAIC2.B.00         dep  25.107  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000
   161   MOSAIC2.B.01         dep  25.120  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000

branches/eam_branches/ipp-20130207/ippconfig/gpc1/psphot.config

@@ -48 +48 @@
   BACKGROUND.YBIN                   S32  32            # size of background superpixels
   DIFF_STATS                        BOOL TRUE          # Calculate diff-specific stats
-  OUTPUT.FORMAT                     STR  PS1_DV2       # Format of output file
+  OUTPUT.FORMAT                     STR  PS1_DV3       # Format of output file
 
   PSPHOT.EXT.NSIGMA.LIMIT           F32  0.0
@@ -56 +56 @@
   PSPHOT.EXT.FIT.ALL.SOURCES        BOOL TRUE
   PSPHOT.EXT.FIT.ALL.THRESH         F32  1000
+# Lower threshold as requested by MOPS
+  EXTENDED_SOURCE_MODELS METADATA 
+    TYPE      EXTENDED_SOURCE_MODEL  MODEL            SNLIM   PSF_CONVOLVED
+    TRAIL_RAW EXTENDED_SOURCE_MODEL  PS_MODEL_TRAIL   4.75    FALSE
+  END
 END
 
@@ -65 +70 @@
   PSPHOT.EXT.FIT.ALL.SOURCES        BOOL TRUE
   PSPHOT.EXT.FIT.ALL.THRESH         F32  1000
+# Lower threshold as requested by MOPS
+  EXTENDED_SOURCE_MODELS METADATA 
+    TYPE      EXTENDED_SOURCE_MODEL  MODEL            SNLIM   PSF_CONVOLVED
+    TRAIL_RAW EXTENDED_SOURCE_MODEL  PS_MODEL_TRAIL   4.75    FALSE
+  END
 END
 

branches/eam_branches/ipp-20130207/ippconfig/recipes/ppStats.config

@@ -358 +358 @@
   ANALYSIS      STR     MASKFRAC_NPIX
 END
+
+# stats for sky calibration (astrometry, summary stats)
+SKYCALSTATS     METADATA
+  HEADER        MULTI
+  HEADER        STR     CERROR
+  HEADER        STR     DT_ASTR  # elapsed time in astrometry processing
+  HEADER        STR     NASTRO
+
+  HEADER        STR     ZPT_OBS
+  HEADER        STR     ZPT_ERR
+  HEADER        STR     FWHM_MAJ
+  HEADER        STR     FWHM_MIN
+
+  HEADER        STR     PSLIB_V
+  HEADER        STR     MODULE_V
+  HEADER        STR     PHOT_V
+  HEADER        STR     ASTRO_V
+  HEADER        STR     STATS_V
+  HEADER        STR     IMAGE_V
+  HEADER        STR     STREAK_V
+  
+  HEADER        STR     NDET            # sources->n
+  HEADER        STR     NDET_EXT
+  HEADER        STR     NDET_FRC
+
+  ANALYSIS      MULTI   # metadata blocks to search in chip/cell/readout->analysis
+  ANALYSIS      STR     PSPHOT.HEADER
+  ANALYSIS      STR     PSASTRO.HEADER
+
+  ANALYSIS      STR     MASKFRAC_REF_STATIC
+  ANALYSIS      STR     MASKFRAC_REF_MAGIC
+  ANALYSIS      STR     MASKFRAC_REF_ADVISORY
+  ANALYSIS      STR     MASKFRAC_REF_DYNAMIC
+  ANALYSIS      STR     MASKFRAC_REF_NPIX
+
+  ANALYSIS      STR     MASKFRAC_MAX_STATIC
+  ANALYSIS      STR     MASKFRAC_MAX_MAGIC
+  ANALYSIS      STR     MASKFRAC_MAX_ADVISORY
+  ANALYSIS      STR     MASKFRAC_MAX_DYNAMIC
+  ANALYSIS      STR     MASKFRAC_MAX_NPIX
+END

branches/eam_branches/ipp-20130207/ippconfig/recipes/ppStatsFromMetadata.config

@@ -337 +337 @@
 
 
+BACKGROUND_CHIP METADATA
+  ENTRY MULTI
+  TYPE   VAL  KEYWORD             TYPE STATISTIC         FLAG 
+  ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag
+
+  # Revision values
+  ENTRY  VAL  PSLIB_V             STR  CONSTANT         -ver_pslib
+  ENTRY  VAL  MODULE_V            STR  CONSTANT         -ver_psmodules
+  ENTRY  VAL  STATS_V             STR  CONSTANT         -ver_ppstats
+
+END
+
 BACKGROUND_WARP METADATA
   ENTRY MULTI
@@ -550 +562 @@
   ENTRY  VAL  ZPT_OBS             F32  CONSTANT         -zpt_obs
   ENTRY  VAL  ZPT_ERR             F32  CONSTANT         -zpt_stdev
+  # don't know why these come out as F64 instead of F32
+  ENTRY  VAL  FWHM_MAJ            F64  CONSTANT         -fhwm_major
+  ENTRY  VAL  FWHM_MIN            F64  CONSTANT         -fhwm_minor
 
   ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag

branches/eam_branches/ipp-20130207/ppStack/src/ppStackCamera.c

@@ -48 +48 @@
     bool convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images before stack?
     bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL");
-    
+    //    fprintf(stderr,"ppSC: %d\n",doBackground);    
     psArray *runImages = pmFPAfileDefineMultipleFromRun(&status, NULL, config,
                                                         "PPSTACK.INPUT"); // Input images from previous run
@@ -218 +218 @@
 
             // Grab bkgmodel information here
-            if (doBackground) {
-              if (!bkgmodel ||  strlen(bkgmodel) == 0) {
+            
+            //      if (doBackground) {
+              if ((!bkgmodel) || (strlen(bkgmodel) == 0)) {
                 // We have no background models.
+                //              fprintf(stderr,"No backgrounds specified, resettting recipe value.\n");
+                doBackground = false;
+                psMetadataAddBool(recipe, PS_LIST_TAIL, "BACKGROUND.MODEL", PS_META_REPLACE,
+                                  "Do photometry on stacked image?", false);
+
               }
               else {
@@ -227 +233 @@
                 if (!inputBKG) {
                   // We failed to generate an pmFPAfile, so disable the background construction and continue.
+                  //              fprintf(stderr,"No backgrounds specified, resettting recipe value.\n");
                   doBackground = false;
                   psMetadataAddBool(recipe, PS_LIST_TAIL, "BACKGROUND.MODEL", PS_META_REPLACE,
@@ -238 +245 @@
                 }
               }// End bkgmodel
-            }
+              //            }
 
 

branches/eam_branches/ipp-20130207/ppStack/src/ppStackCombineAlternate.c

@@ -88 +88 @@
   }
 
+  if (!ppStackLinearScale(inputs, config)) {
+    psFree(inputs);
+    return(false);
+  }
 
   // Do combination

branches/eam_branches/ipp-20130207/ppStack/src/ppStackLoop.c

@@ -63 +63 @@
     }
 
-    bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL");
+    //    bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL");
 
     // Prepare for combination
-    if (!ppStackCombinePrepare("PPSTACK.OUTPUT", "PPSTACK.OUTPUT.EXP", doBackground ? "PPSTACK.OUTPUT.BKGMODEL" : NULL, PPSTACK_FILES_STACK, stack, options, config)) {
+    if (!ppStackCombinePrepare("PPSTACK.OUTPUT", "PPSTACK.OUTPUT.EXP", options->doBackground ? "PPSTACK.OUTPUT.BKGMODEL" : NULL, PPSTACK_FILES_STACK, stack, options, config)) {
         psError(psErrorCodeLast(), false, "Unable to prepare for combination.");
         psFree(stack);
@@ -146 +146 @@
 
     // Generate median background stack here.
-    if (doBackground && !ppStackCombineBackground(stack, options, config)) {
+    if (options->doBackground && !ppStackCombineBackground(stack, options, config)) {
       psError(psErrorCodeLast(), false, "Unable to generate median of background images.");
       psFree(stack);

branches/eam_branches/ipp-20130207/ppStack/src/ppStackOptions.c

@@ -43 +43 @@
     options->matchZPs = true;
     options->photometry = false;
+    options->doBackground = false;
     options->stats = NULL;
     options->statsFile = NULL;

branches/eam_branches/ipp-20130207/ppStack/src/ppStackOptions.h

@@ -8 +8 @@
     bool matchZPs;                      // Adjust relative fluxes based on transparency analysis?
     bool photometry;                    // Perform photometry?
+    bool doBackground;                  // Do background model combination?
     psMetadata *stats;                  // Statistics for output
     FILE *statsFile;                    // File to which to write statistics

branches/eam_branches/ipp-20130207/ppStack/src/ppStackSetup.c

@@ -17 +17 @@
 
     options->convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images?
+    options->doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL");
+
     if (!psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) {
         psWarning("No PSFs provided --- unable to convolve to common PSF.");

branches/eam_branches/ipp-20130207/psLib/src/imageops/psImageInterpolate.c

@@ -47 +47 @@
                              4,    // LANCZOS2
                              6,    // LANCZOS3
-                             8     // LANCZOS4
+                             8,    // LANCZOS4
+                             2     // BILINEAR_SIMPLE
 };
 

branches/eam_branches/ipp-20130207/psModules/src/imcombine/pmStack.c

@@ -1418 +1418 @@
       }
       combined->image->data.F32[y][x] = stats->robustMedian;
-      combined->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0;
+      if (combined->mask) {
+        combined->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0;
+      }
 #if (0)
       if ((x == 59)&&(y > 40)&&(y < 50)) {

branches/eam_branches/ipp-20130207/psModules/src/objects/models/pmModel_TRAIL.c

@@ -256 +256 @@
     if (!isfinite(axes.theta)) return false;
 
-    float size = (axes.major > source->moments->Mrf) ? axes.major : source->moments->Mrf;
+    float size = (axes.major > sqrt(source->moments->Mrf)) ? axes.major : sqrt(source->moments->Mrf);
+    //    if (size > psfAxes.major) { size -= psfAxes.major; }
+    //else { size = psfAxes.major; }
 
     // axes.major is a sigma in the major direction; scale to 

branches/eam_branches/ipp-20130207/psphot/src/psphotSourceSize.c

@@ -423 +423 @@
     psLogMsg("psModules.objects", PS_LOG_INFO, "Source Size classifications: %4s %4s %4s %4s %4s", "Npsf", "Next", "Nsat", "Ncr", "Nskip");
     // Determine if this readout is above the threshold to ext fit all sources
+    options->extFitAllReadout = false;
     if (options->extFitAll) {
       float maskFrac = psMetadataLookupF32(&status,readout->analysis,"READOUT.MASK.FRAC");
@@ -584 +585 @@
         // the EXT_LIMIT flag
         if (options->extFitAllReadout) {
+          psTrace("psphotSourceClassRegion.EXTALT",10,"In extFitAll: %d %d\n",options->extFitAll,options->extFitAllReadout);
           source->mode |= PM_SOURCE_MODE_EXT_LIMIT;
         }

branches/eam_branches/ipp-20130207/tools/fixburntool

@@ -106 +106 @@
 
 {
-    my $command = "$burntool --continue 10 --exp_id $exp_id --class_id $class_id --this_uri $uri --previous_uri $prev_uri";
+    my $command = "$burntool --exp_id $exp_id --class_id $class_id --this_uri $uri --previous_uri $prev_uri";
     print "$command\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =