Changeset 28334 for branches/czw_branch/20100519

Timestamp:

Jun 15, 2010, 11:22:56 AM (16 years ago)

Author:

watersc1

Message:

association and jpegs look fine. random simtest errors are a bit worrying, though.

Location:

branches/czw_branch/20100519

Files:

• . (modified) (1 prop)
• Ohana (modified) (1 prop)
• Ohana/src/photdbc/src/copy_images.c (modified) (1 diff)
• Ohana/src/photdbc/src/join_stars.c (modified) (2 diffs)
• Ohana/src/photdbc/src/make_subcatalog.c (modified) (2 diffs)
• Ohana/src/photdbc/src/photdbc.c (modified) (1 diff)
• dbconfig/changes.txt (modified) (2 diffs)
• dbconfig/pstamp.md (modified) (1 diff)
• ippScripts/scripts/addstar_run.pl (modified) (5 diffs)
• ippScripts/scripts/minidvodb_createdb.pl (modified) (4 diffs)
• ippScripts/scripts/skycell_jpeg.pl (modified) (2 diffs)
• ippTasks/diff.pro (modified) (2 diffs)
• ippTasks/minidvodb.pro (modified) (4 diffs)
• ippTasks/stack.pro (modified) (6 diffs)
• ippToPsps/src/ippToPspsBatchDetection.c (modified) (5 diffs)
• ippTools/share/difftool_addsummary.sql (added)
• ippTools/share/difftool_tosummary.sql (added)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff)
• ippTools/share/stacktool_addsummary.sql (added)
• ippTools/share/stacktool_associationdefine_select.sql (added)
• ippTools/share/stacktool_sassskyfile.sql (added)
• ippTools/share/stacktool_tosummary.sql (added)
• ippTools/share/warptool_addsummary.sql (added)
• ippTools/share/warptool_tosummary.sql (added)
• ippTools/src/addtool.c (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (6 diffs)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff)
• ippTools/src/stacktool.c (modified) (2 diffs)
• ippconfig/recipes/filerules-mef.mdc (modified) (2 diffs)
• ippconfig/recipes/filerules-simple.mdc (modified) (2 diffs)
• ippconfig/recipes/filerules-split.mdc (modified) (2 diffs)
• ppBackground (modified) (1 prop)
• ppBackground/configure.ac (modified) (1 diff)
• ppBackground/src (modified) (1 prop)
• ppBackground/src/Makefile.am (modified) (4 diffs)
• ppBackground/src/ppBackground.c (modified) (3 diffs)
• ppBackground/src/ppBackground.h (modified) (2 diffs)
• ppBackground/src/ppBackgroundArguments.c (modified) (3 diffs)
• ppBackground/src/ppBackgroundCamera.c (modified) (3 diffs)
• ppBackground/src/ppBackgroundExit.c (modified) (1 diff)
• ppBackground/src/ppBackgroundLoop.c (modified) (9 diffs)
• ppBackground/src/ppBackgroundModel.c (deleted)
• ppBackground/src/ppBackgroundRestore.c (copied) (copied from trunk/ppBackground/src/ppBackgroundRestore.c )
• ppBackground/src/ppBackgroundVersionDefinitions.h.in (modified) (1 diff)
• ppImage/src/ppImageLoop.c (modified) (6 diffs)
• ppImage/src/ppImageMosaic.c (modified) (5 diffs)
• ppStack/src/ppStackTarget.c (modified) (2 diffs)
• psLib/src/sys/psAbort.c (modified) (3 diffs)
• psLib/src/sys/psThread.c (modified) (1 diff)
• psModules/src/config/pmConfig.c (modified) (3 diffs)
• pstamp/src/pstamp.h (modified) (1 diff)
• tools/chip_outputs.pl (modified) (1 diff)

branches/czw_branch/20100519/Ohana/src/photdbc/src/copy_images.c

@@ -3 +3 @@
 int copy_images (char *outdir) {
 
-  int status, Nimage;
+  int status;
+  off_t Nimage;
   char *ImageOut;
   FITS_DB in;

branches/czw_branch/20100519/Ohana/src/photdbc/src/join_stars.c

@@ -7 +7 @@
   off_t Naverage, Nmeasure, *index;
   double *X, *Y, dX, dY, dR, RADIUS2;
-  double Sr, Sd, Rmid, Dmid;
+  double Rmid, Dmid;
   int basecode, baseNsec, Nsecfilt, *found;
 
@@ -202 +202 @@
 
   if (Nmeas != Nmeasure) {
-    fprintf (stderr, "failure to match %d measures (%d of %d matched)\n", Nmeasure - Nmeas, Nmeas, Nmeasure);
+    fprintf (stderr, "failure to match "OFF_T_FMT" measures ("OFF_T_FMT" of "OFF_T_FMT" matched)\n", Nmeasure - Nmeas, Nmeas, Nmeasure);
   }
   

branches/czw_branch/20100519/Ohana/src/photdbc/src/make_subcatalog.c

@@ -5 +5 @@
 int make_subcatalog (Catalog *subcatalog, Catalog *catalog) {
   
-  int i, j, offset;
-  int NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm, Nsecfilt;
+  off_t i, j, offset;
+  off_t NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm, Nsecfilt;
   double mag, minMag;
 
@@ -105 +105 @@
 
   if (VERBOSE) {
-    fprintf (stderr, "%d: using %d stars (%d measures) for catalog\n", i, 
+    fprintf (stderr, OFF_T_FMT": using "OFF_T_FMT" stars ("OFF_T_FMT" measures) for catalog\n", i, 
              subcatalog[0].Naverage, subcatalog[0].Nmeasure);
   }

branches/czw_branch/20100519/Ohana/src/photdbc/src/photdbc.c

@@ -60 +60 @@
         
     // XXX add other filters here:
-    join_stars (&outcatalog);
+    // join_stars (&outcatalog);
     // unique_measures (catalog);
     // flag_measures (&db, catalog);

branches/czw_branch/20100519/dbconfig/changes.txt

@@ -1756 +1756 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
--- Version 1.1.69
+ALTER TABLE pstampRequest ADD COLUMN timestamp TIMESTAMP AFTER outdir;
+
+-- Version 1.1.69 warp/stack/diff summary and stack associations.
+
 CREATE TABLE warpSummary (
     warp_id     BIGINT,
@@ -1804 +1807 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+
+
+
+

branches/czw_branch/20100519/dbconfig/pstamp.md

@@ -31 +31 @@
     uri         STR         255
     outdir      STR         255
+    timestamp   UTC         0001-01-01T00:00:00Z
     fault       S32         0
 END

branches/czw_branch/20100519/ippScripts/scripts/addstar_run.pl

@@ -36 +36 @@
     exit($PS_EXIT_CONFIG_ERROR);
 }
-
+my $minidvodb_path;
 my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
      $no_op, $redirect, $save_temps);
@@ -132 +132 @@
                     &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
                 my $comp = $$components[0];
+                $minidvodb_path = $comp->{minidvodb_path};
                 $minidvodb_name = $comp->{minidvodb_name};
         
+                if (!defined($minidvodb_path)) {
+                    &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
+                } 
                 if (!defined($minidvodb_name)) {
                     &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
-                } 
-                
+                }
             }
             # tack on the minidvodb part to the db.
-            $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
+#           $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
+#we don't need this now that I fixed the paths     
             
         }
@@ -160 +164 @@
         my $command  = "$addstar -update"; # XXX optionally set -update?
         $command .= " -D CAMERA $camdir";
-        $command .= " -D CATDIR $dvodbReal";
+        $command .= " -D CATDIR $minidvodb_path";
         $command .= " $realFile";
         $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference
@@ -182 +186 @@
 $fpaCommand .= " -path_base $outroot";
 $fpaCommand .= " -dtime_addstar $dtime_addstar";
-$fpaCommand .= " -dvodb_path $dvodbReal" if defined $dvodbReal;
+$fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
 $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
 $fpaCommand .= " -dbname $dbname" if defined $dbname;
@@ -213 +217 @@
         $command .= " -addprocessedexp";
         $command .= " -fault $exit_code";
-        $command .= " -dvodb_path $dvodbReal" if defined $dvodbReal;
+        $command .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
         $command .= " -path_base $outroot" if defined $outroot;
         $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));

branches/czw_branch/20100519/ippScripts/scripts/minidvodb_createdb.pl

@@ -39 +39 @@
 }
 
-my (  $outroot, $dbname, $dvodb,$minidvodb_interval, $minidvodb_group, $camera,  $verbose, $no_update,
+my (  $outroot, $dbname, $dvodb, $minidvodb,$minidvodb_interval, $minidvodb_group, $camera,  $verbose, $no_update,
      $no_op, $redirect, $save_temps);
 GetOptions(    
@@ -47 +47 @@
     'outroot|w=s'       => \$outroot, # output file base name
     'dvodb|w=s'         => \$dvodb,  # output DVO database
+    'minidvodb|w=s'     => \$minidvodb, # output miniDVODB
     'interval|w=s'      => \$minidvodb_interval, #interval between creation of minidvodbs (default = 1day)
     'verbose'           => \$verbose,   # Print to stdout
@@ -97 +98 @@
     exit(4);
 }
+my $minidvodbReal;
+if (defined $minidvodb) {
+    $minidvodbReal = $ipprc->dvo_catdir( $minidvodb ); # catdir for DVO
+    $minidvodbReal = $ipprc->convert_filename_absolute( $minidvodbReal );
+} else {
+    warn("minidvodb undefined:\n");
+    exit(4);
+}
+
 
 if (!defined $minidvodb_interval) {
@@ -215 +225 @@
     my $fpaCommand = "$addtool -addminidvodbrun";
     $fpaCommand .= " -set_minidvodb_group $minidvodb_group";
+    $fpaCommand .= " -set_minidvodb_path  $minidvodbReal" if defined $minidvodbReal;
     $fpaCommand .= " -set_mergedvodb_path $dvodbReal";
     $fpaCommand .= " -dbname $dbname" if defined $dbname;

branches/czw_branch/20100519/ippScripts/scripts/skycell_jpeg.pl

@@ -273 +273 @@
         my $path_base  = $imfile->{path_base};
         my $quality    = $imfile->{quality};
-        my $state      = $imfile->{data_state};
+        my $state      = $imfile->{state};
         my $fault      = $imfile->{fault};
         my $projection_cell = $skycell_id;
-        if ($quality == 8007 or $state ne 'full' or $fault != 0) {
+        if ($quality == 8007 or $state ne 'full') {
             next;
         }
@@ -333 +333 @@
     if (scalar (keys %tangents) == 0) {
         my $projection_cell = 'NULL';
-        $command = "$stacktool -addsummary -stack_id $stage_id -projection_cell $projection_cell -path_base $outroot";
-        if (defined($dbname)) {
-            $command .= " -dbname $dbname";
-        }
-        
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("unable to update diff summary: $error_code", $stage_id, $error_code);
+        $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot";
+        if (defined($dbname)) {
+            $command .= " -dbname $dbname";
+        }
+#       print "$command\n";
+        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("unable to update stack summary: $error_code", $stage_id, $error_code);
         }
     }   

branches/czw_branch/20100519/ippTasks/diff.pro

@@ -574 +574 @@
     book getword diffPendingSummary $pageName workdir -var WORKDIR_TEMPLATE
     book getword diffPendingSummary $pageName dbname -var DBNAME
-    book getword warpPendingSummary $pageName tess_id -var TESS_DIR
+    book getword diffPendingSummary $pageName tess_id -var TESS_DIR
     book getword diffPendingSummary $pageName diff_mode -var DIFF_MODE
     book getword diffPendingSummary $pageName state -var RUN_STATE
@@ -601 +601 @@
 
     ## generate outroot specific to this exposure
-    sprintf outroot "%s/%s/%s/%s.%s.%sdif.%s.summary" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_TAG $DIFF_ID
+    sprintf outroot "%s/%s/%s.%sdif.%s.summary" $WORKDIR $TESS_ID $TESS_ID $DIFF_TAG $DIFF_ID
 
 

branches/czw_branch/20100519/ippTasks/minidvodb.pro

@@ -131 +131 @@
 ## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run.
 macro add.minidvodb
-  if ($0 != 5)
-    echo "USAGE: add.minidvodb (minidvodb_group) (dvodb) (interval days) (camera)"
+  if ($0 != 6)
+    echo "USAGE: add.minidvodb (minidvodb_group) (minidvodb) (dvodb) (interval days) (camera)"
     break
   end
@@ -139 +139 @@
   book newpage MINIDVODB $1
   book setword MINIDVODB $1 MINIDVODB_GROUP $1
-  book setword MINIDVODB $1 DVODB $2
+  book setword MINIDVODB $1 DVODB $3
   book setword MINIDVODB $1 STATE PENDING
   #merge  
   book newpage MINIDVODB_MERGE $1
   book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1
-  book setword MINIDVODB_MERGE $1 DVODB $2
+  book setword MINIDVODB_MERGE $1 DVODB $3
   book setword MINIDVODB_MERGE $1 STATE PENDING
   #active  
   book newpage MINIDVODB_ACTIVE $1
   book setword MINIDVODB_ACTIVE $1 MINIDVODB_GROUP $1
-  book setword MINIDVODB_ACTIVE $1 DVODB $2
+  book setword MINIDVODB_ACTIVE $1 DVODB $3
   book setword MINIDVODB_ACTIVE $1 STATE PENDING
 
@@ -155 +155 @@
   book newpage MINIDVODB_CREATE $1
   book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1
-  book setword MINIDVODB_CREATE $1 DVODB $2
-  book setword MINIDVODB_CREATE $1 DVODB_DAYS $3
-  book setword MINIDVODB_CREATE $1 CAMERA $4  
+  book setword MINIDVODB_CREATE $1 MINIDVODB $2
+  book setword MINIDVODB_CREATE $1 DVODB $3
+  book setword MINIDVODB_CREATE $1 DVODB_DAYS $4
+  book setword MINIDVODB_CREATE $1 CAMERA $5  
   book setword MINIDVODB_CREATE $1 STATE PENDING  
 end
@@ -542 +543 @@
      
     book setword MINIDVODB_CREATE $minidvodb_group STATE DONE
+    book getword MINIDVODB_CREATE $minidvodb_group MINIDVODB -var minidvodb 
     book getword MINIDVODB_CREATE $minidvodb_group DVODB -var dvodb 
     book getword MINIDVODB_CREATE $minidvodb_group DVODB_DAYS -var dvodb_days
     book getword MINIDVODB_CREATE $minidvodb_group CAMERA -var camera
-    $run = minidvodb_createdb.pl --camera $camera --outroot $dvodb --dvodb $dvodb --minidvodb_group $minidvodb_group --interval $dvodb_days
+    $run = minidvodb_createdb.pl --camera $camera --outroot $dvodb --dvodb $dvodb --minidvodb $minidvodb --minidvodb_group $minidvodb_group --interval $dvodb_days
  
     if ($DB:n == 0)

branches/czw_branch/20100519/ippTasks/stack.pro

@@ -10 +10 @@
 ### Initialise the books containing the tasks to do
 book init stackSumSkyfile
+book init stackPendingSummary
 #book init stackCleanup
 
@@ -16 +17 @@
 $stack_revert_DB = 0
 #$stackCleanup_DB = 0
+$stackSummary_DB = 0
 
 ### Check status of stacking tasks
@@ -26 +28 @@
 macro stack.reset
   book init stackSumSkyfile
+  book init stackPendingSummary
 #  book init stackCleanup
 end
@@ -40 +43 @@
     active false
   end
+  task stack.summary.load
+    active true
+  end
+  task stack.summary.run
+    active true
+  end
 end
 
@@ -51 +60 @@
   end
   task stack.revert
+    active false
+  end
+  task stack.summary.load
+    active false
+  end
+  task stack.summary.run
     active false
   end
@@ -403 +418 @@
 end
 
+### Load tasks for doing the stack summary
+### Tasks are loaded into stackPendingSummary
+task           stack.summary.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 1200
+  npending     1
+
+  stdout       NULL
+  stderr       $LOGDIR/stack.summary.load
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = stacktool -tosummary
+    if ($DB:n == 0) 
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$stackSummary_DB
+      $run = $run -dbname $DB:$stackSummary_DB
+      $stackSummary_DB ++
+      if ($stackSummary_DB >= $DB:n) set stackSummary_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit     0
+    # convert 'stdout' to book format
+    ipptool2book stdout stackPendingSummary -key sass_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook stackPendingSummary
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup stackPendingSummary
+  end
+
+  # locked list
+  task.exit     default
+    showcommand failure
+  end
+
+  task.exit     crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit     timeout
+    showcommand timeout
+  end
+end
+
+### Run tasks for doing the stack summary
+task           stack.summary.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages stackPendingSummary -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+
+    # look for new images in stackPendingSummary (pantaskState == INIT)
+    book getpage stackPendingSummary 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword stackPendingSummary $pageName pantaskState RUN
+    book getword stackPendingSummary $pageName sass_id -var SASS_ID
+    book getword stackPendingSummary $pageName camera -var CAMERA
+    book getword stackPendingSummary $pageName workdir -var WORKDIR_TEMPLATE
+    book getword stackPendingSummary $pageName dbname -var DBNAME
+    book getword stackPendingSummary $pageName tess_id -var TESS_DIR
+    book getword stackPendingSummary $pageName state -var RUN_STATE
+    book getword stackPendingSummary $pageName projection_cell -var projection_cell
+
+    # set the host and workdir
+    host anyhost
+    strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR
+
+    basename $TESS_DIR -var TESS_ID
+
+    ## generate outroot specific to this association
+    sprintf outroot "%s/%s/%s.stk.%s.summary" $WORKDIR $TESS_ID $TESS_ID $SASS_ID
+
+    stdout $LOGDIR/stack.summary.log
+    stderr $LOGDIR/stack.summary.log
+
+    $run = skycell_jpeg.pl --stage stack --stage_id $SASS_ID --camera $CAMERA --outroot $outroot
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+        echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit       default
+    process_exit stackPendingSummary $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit       crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword stackPendingSummary $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit       timeout
+    showcommand timeout
+    book setword stackPendingSummary $options:0 pantaskState TIMEOUT
+  end
+end
+

branches/czw_branch/20100519/ippToPsps/src/ippToPspsBatchDetection.c

@@ -54 +54 @@
     float zptObs, zeroPoint, exposureTime;
     char filterType[20];
+    double obsTime;
+    double expStart;
+    double expTime;
     status=0; fits_read_key(fitsIn, TFLOAT, "ZPT_OBS", &zptObs, NULL, &status);
     status=0; fits_read_key(fitsIn, TFLOAT, "ZPT_REF", &zeroPoint, NULL, &status);
     status=0; fits_read_key(fitsIn, TFLOAT, "EXPREQ", &exposureTime, NULL, &status);
     status=0; fits_read_key(fitsIn, TSTRING, "FILTERID", filterType, NULL, &status);
-    
+    status=0; fits_read_key(fitsIn, TDOUBLE, "MJD-OBS", &expStart, NULL, &status);
+    status=0; fits_read_key(fitsIn, TDOUBLE, "EXPTIME", &expTime, NULL, &status);
+    obsTime = expStart + (expTime/172800); // exp start plus half exp time (converted from secs to days)
+   
     ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, 1, "FrameMeta", true);
 
@@ -83 +89 @@
 
     // stuff to keep from psf.hdr header
-    double obsTime = 0.0;
     int sourceId = -1;
     int imageId = -1;
@@ -132 +137 @@
     long ippDetectID[MAXDETECT];
     long imageID[MAXDETECT];
-    float obsTimes[MAXDETECT];
+    double obsTimes[MAXDETECT];
     float instFlux[MAXDETECT];
     float instFluxErr[MAXDETECT];
@@ -195 +200 @@
             status=0; fits_read_key(fitsIn, TFLOAT, "IQ_FW2", &momentMin, NULL, &status);
             status=0; fits_read_key(fitsIn, TLONG, "IMAGEID", &imageId, NULL, &status);
-            status=0; fits_read_key(fitsIn, TDOUBLE, "MJD-OBS", &obsTime, NULL, &status);
             status=0; fits_read_key(fitsIn, TLONG, "SOURCEID", &sourceId, NULL, &status);
 
@@ -333 +337 @@
             fits_write_col(this->fitsOut, TBYTE, DETECTION_SURVEYID, 1, 1, nDet, surveyIDs, &status);
             fits_write_col(this->fitsOut, TLONGLONG, DETECTION_IMAGEID, 1, 1, nDet, imageID, &status);
-            fits_write_col(this->fitsOut, TFLOAT, DETECTION_OBSTIME, 1, 1, nDet, obsTimes, &status);
+            fits_write_col(this->fitsOut, TDOUBLE, DETECTION_OBSTIME, 1, 1, nDet, obsTimes, &status);
             fits_write_col(this->fitsOut, TFLOAT, DETECTION_INSTFLUX, 1, 1, nDet, instFlux, &status);
             fits_write_col(this->fitsOut, TFLOAT, DETECTION_INSTFLUXERR, 1, 1, nDet, instFluxErr, &status);

branches/czw_branch/20100519/ippTools/share/pxadmin_create_tables.sql

@@ -1411 +1411 @@
         uri VARCHAR(255),
         outdir     VARCHAR(255),
+        timestamp TIMESTAMP,
         fault SMALLINT,
         PRIMARY KEY(req_id),

branches/czw_branch/20100519/ippTools/src/addtool.c

@@ -817 +817 @@
   }
 
+  if (minidvodb_path) {
+    psStringAppend(&minidvodb_path,"/%s",minidvodb_name);
+  }
+
+
   psString query = NULL;
 

branches/czw_branch/20100519/ippTools/src/pstamptool.c

@@ -271 +271 @@
         uri,
         NULL,   // outdir
+        NULL,   // timestamp
         0       // fault
         )) {
@@ -547 +548 @@
     PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault",    false);
 
+    if (!state && !label && !outProduct && !fault && !uri && !outdir && !name && !reqType && !clearfault) {
+        psError(PS_ERR_UNKNOWN, true, "at least one set option is required");
+        return false;
+    }
+
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-req_id",     "req_id", "==");
@@ -552 +558 @@
     PXOPT_COPY_S32(config->args, where, "-fault",      "fault", "==");
     PXOPT_COPY_STR(config->args, where, "-state",      "state", "==");
+    PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">=");
+    PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<=");
     pxAddLabelSearchArgs(config, where, "-label",      "pstampRequest.label", "LIKE");
     if (!psListLength(where->list)) {
@@ -559 +567 @@
     }
 
-    psString query = psStringCopy("UPDATE pstampRequest SET");
+    psString query = psStringCopy("UPDATE pstampRequest SET timestamp = UTC_TIMESTAMP()");
 
     psString stateCheck = NULL;
-    char c = ' ';
     if (state) {
-        psStringAppend(&query, "%c state = '%s'", c, state);
-        c = ',';
+        psStringAppend(&query, ", state = '%s'", state);
         if (!strcmp(state, "goto_cleaned")) {
             psStringAppend(&stateCheck, " AND state != 'cleaned'");
@@ -571 +577 @@
     }
     if (label) {
-        psStringAppend(&query, "%c label = '%s'", c, label);
-        c = ',';
+        psStringAppend(&query, ", label = '%s'", label);
     }
     if (outProduct) {
-        psStringAppend(&query, "%c outProduct = '%s'", c, outProduct);
-        c = ',';
+        psStringAppend(&query, ", outProduct = '%s'", outProduct);
     }
     if (outdir) {
-        psStringAppend(&query, "%c outdir = '%s'", c, outdir);
-        c = ',';
+        psStringAppend(&query, ", outdir = '%s'", outdir);
     }
     if (clearfault) {
@@ -587 +590 @@
             return false;
         }
-        psStringAppend(&query, "%c fault = 0", c);
-        c = ',';
+        psStringAppend(&query, ", fault = 0");
     } else if (fault) {
-        psStringAppend(&query, "%c fault = %d", c, fault);
-        c = ',';
+        psStringAppend(&query, ", fault = %d", fault);
     }
     if (uri) {
-        psStringAppend(&query, "%c uri = '%s'", c, uri);
-        c = ',';
+        psStringAppend(&query, ", uri = '%s'", uri);
     }
     if (name) {
-        psStringAppend(&query, "%c name = '%s'", c, name);
-        c = ',';
+        psStringAppend(&query, ", name = '%s'", name);
     }
     if (reqType) {
-        psStringAppend(&query, "%c reqType = '%s'", c, reqType);
-        c = ',';
-    }
-    if (c != ',') {
-        psError(PS_ERR_UNKNOWN, true, "at least one set option is required");
-        return false;
+        psStringAppend(&query, ", reqType = '%s'", reqType);
     }
 

branches/czw_branch/20100519/ippTools/src/pstamptoolConfig.c

@@ -103 +103 @@
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0,        "search by state", NULL);
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
+    psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL);
+    psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_end", 0, "search by timestamp (<=)", NULL);
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_state", 0,        "new state", NULL);
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_label", 0,        "new label", NULL);

branches/czw_branch/20100519/ippTools/src/stacktool.c

@@ -1372 +1372 @@
   if (psListLength(where->list)) {
     psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-    psStringAppend(&query, " WHERE %s", whereClause);
+    psStringAppend(&query, " AND %s", whereClause);
     psFree(whereClause);
   } else if (!all) {
@@ -1387 +1387 @@
     psFree(limitString);
   }
-  
+
   if (!p_psDBRunQuery(config->dbh, query)) {
     psError(PS_ERR_UNKNOWN, false, "database error");

branches/czw_branch/20100519/ippconfig/recipes/filerules-mef.mdc

@@ -153 +153 @@
 PPBGRESTORE.INPUT       INPUT    @FILES        CHIP       IMAGE
 PPBGRESTORE.BACKMDL     INPUT    @FILES        CHIP       IMAGE
+
+PPBACKGROUND.IMAGE        INPUT    @FILES        CHIP       IMAGE
+PPBACKGROUND.MASK         INPUT    @FILES        CHIP       MASK
+PPBACKGROUND.PATTERN      INPUT    @FILES        CHIP       PATTERN
+PPBACKGROUND.BACKGROUND   INPUT    @FILES        CHIP       IMAGE
 
 ### output file definitions
@@ -330 +335 @@
 
 PPBGRESTORE.OUTPUT      OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     COMP_IMG   CHIP       TRUE      NONE
+
+PPBACKGROUND.OUTPUT          OUTPUT {OUTPUT}.{CHIP.NAME}.fits    IMAGE     COMP_IMG   CHIP       TRUE      NONE
+PPBACKGROUND.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK      COMP_MASK  CHIP       TRUE      NONE

branches/czw_branch/20100519/ippconfig/recipes/filerules-simple.mdc

@@ -128 +128 @@
 
 PPVIZPATTERN.INPUT        INPUT    @FILES        CHIP       PATTERN
+
+PPBACKGROUND.IMAGE        INPUT    @FILES        CHIP       IMAGE
+PPBACKGROUND.MASK         INPUT    @FILES        CHIP       MASK
+PPBACKGROUND.PATTERN      INPUT    @FILES        CHIP       PATTERN
+PPBACKGROUND.BACKGROUND   INPUT    @FILES        CHIP       IMAGE
 
 ### output file definitions
@@ -295 +300 @@
 
 PPBGRESTORE.OUTPUT           OUTPUT {OUTPUT}.fits                 IMAGE           NONE       FPA        TRUE      NONE
+
+PPBACKGROUND.OUTPUT          OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE           NONE       FPA        TRUE      NONE
+PPBACKGROUND.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits      MASK            NONE       FPA        TRUE      NONE

branches/czw_branch/20100519/ippconfig/recipes/filerules-split.mdc

@@ -139 +139 @@
                           
 PPVIZPATTERN.INPUT        INPUT    @FILES        CHIP       PATTERN
+
+PPBACKGROUND.IMAGE        INPUT    @FILES        CHIP       IMAGE
+PPBACKGROUND.MASK         INPUT    @FILES        CHIP       MASK
+PPBACKGROUND.PATTERN      INPUT    @FILES        CHIP       PATTERN
+PPBACKGROUND.BACKGROUND   INPUT    @FILES        CHIP       IMAGE
 
 ### output file definitions
@@ -321 +326 @@
 PPVIZPATTERN.CHIP            OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits      IMAGE           NONE       CHIP       TRUE      NONE
 
+PPBACKGROUND.OUTPUT          OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE           COMP_IMG   CHIP       TRUE      NONE
+PPBACKGROUND.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits      MASK            COMP_MASK  CHIP       TRUE      NONE
 
 # FILERULE naming operators:

branches/czw_branch/20100519/ppBackground/configure.ac

@@ -20 +20 @@
 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0])
 PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0])
-PKG_CHECK_MODULES([PSPHOT], [psphot >= 1.0.0])
+PKG_CHECK_MODULES([PPSTATS], [ppStats >= 1.0.0]) 
+PKG_CHECK_MODULES([PSPHOT], [psphot])
+
+AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing])
+if test "$ERRORCODES" = "missing" ; then
+  AC_MSG_ERROR([psParseErrorCodes is required])
+fi
 
 dnl Set CFLAGS for build

branches/czw_branch/20100519/ppBackground/src/Makefile.am

@@ -2 +2 @@
 
 if HAVE_SVNVERSION
-PPVIZPATTERN_VERSION=`$(SVNVERSION) ..`
+PPBACKGROUND_VERSION=`$(SVNVERSION) ..`
 else
-PPVIZPATTERN_VERSION="UNKNOWN"
+PPBACKGROUND_VERSION="UNKNOWN"
 endif
 
 if HAVE_SVN
-PPVIZPATTERN_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
-PPVIZPATTERN_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+PPBACKGROUND_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPBACKGROUND_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
 else
-PPVIZPATTERN_BRANCH="UNKNOWN"
-PPVIZPATTERN_SOURCE="UNKNOWN"
+PPBACKGROUND_BRANCH="UNKNOWN"
+PPBACKGROUND_SOURCE="UNKNOWN"
 endif
 
@@ -19 +19 @@
 ppBackgroundVersionDefinitions.h: ppBackgroundVersionDefinitions.h.in FORCE
         -$(RM) ppBackgroundVersionDefinitions.h
-        $(SED) -e "s|@PPVIZPATTERN_VERSION@|\"$(PPVIZPATTERN_VERSION)\"|" -e "s|@PPVIZPATTERN_BRANCH@|\"$(PPVIZPATTERN_BRANCH)\"|" -e "s|@PPVIZPATTERN_SOURCE@|\"$(PPVIZPATTERN_SOURCE)\"|" ppBackgroundVersionDefinitions.h.in > ppBackgroundVersionDefinitions.h
+        $(SED) -e "s|@PPBACKGROUND_VERSION@|\"$(PPBACKGROUND_VERSION)\"|" -e "s|@PPBACKGROUND_BRANCH@|\"$(PPBACKGROUND_BRANCH)\"|" -e "s|@PPBACKGROUND_SOURCE@|\"$(PPBACKGROUND_SOURCE)\"|" ppBackgroundVersionDefinitions.h.in > ppBackgroundVersionDefinitions.h
 FORCE: ;
 
@@ -25 +25 @@
 
 
-ppBackground_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PPBACKGROUND_CFLAGS)
-ppBackground_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PSPHOT_LIBS)   $(PPBACKGROUND_LIBS)
+ppBackground_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPBACKGROUND_CFLAGS)
+ppBackground_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSTATS_LIBS)   $(PSPHOT_LIBS)   $(PPBACKGROUND_LIBS)
 
 ppBackground_SOURCES =          \
@@ -35 +35 @@
         ppBackgroundLoop.c      \
         ppBackgroundVersion.c   \
-        ppBackgroundModel.c     \
-        ppBackgroundErrorCodes.c
+        ppBackgroundRestore.c   \
+        ppBackgroundErrorCodes.c        \
+        ppBackgroundExit.c
 
 noinst_HEADERS = \

branches/czw_branch/20100519/ppBackground/src/ppBackground.c

@@ -6 +6 @@
 #include <pslib.h>
 #include <psmodules.h>
+#include <psphot.h>
 
 #include "ppBackground.h"
@@ -12 +13 @@
 {
     ppBackgroundVersionPrint();
+
+    pmErrorRegister();
+    psphotErrorRegister();
+    ppBackgroundErrorRegister();
 
     ppBackgroundData *data = ppBackgroundDataInit(&argc, argv);
@@ -46 +51 @@
         }
         data->statsFile = NULL;
-        pmConfigRunFilenameAddWrite(data->config, "STATS", data->statsName);
         exitValue = ppBackgroundExitCode(exitValue);
     }

branches/czw_branch/20100519/ppBackground/src/ppBackground.h

@@ -5 +5 @@
 #include <psmodules.h>
 
-#include <ppBackgroundErrorCodes.h>
+#include "ppBackgroundErrorCodes.h"
 
 #define PPBACKGROUND_RECIPE "PPBACKGROUND"      // Recipe name
@@ -37 +37 @@
     );
 
+/// Restore the background to an image
+bool ppBackgroundRestore(
+    pmChip *chip,                       // Chip to correct
+    const pmChip *background,           // Chip with background model
+    const pmChip *pattern,              // Chip with pattern
+    const pmFPAview *view,              // View to data
+    pmConfig *config                    // Configuration
+    );
+
 /// Determine exit code
 psExit ppBackgroundExitCode(

branches/czw_branch/20100519/ppBackground/src/ppBackgroundArguments.c

@@ -45 +45 @@
     assert(data->config);
 
-    psMetadata *arguments = psMetadataAlloc(); // Command-line arguments
+    psMetadata *arguments = data->config->arguments;
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-pattern", 0, "Filename of pattern correction", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-background", 0, "Filename of background model", NULL);
-    psMetadataAddStr(arguments, PS_LIST_TAIL, "-image", 0, "Filename of image", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-image", 0, "Filename of image (required)", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask", 0, "Filename of mask", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "Output statistics file", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-dumpconfig", 0, "Output configuration file", NULL);
     if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 2) {
         usage(argv[0], arguments, data);
@@ -59 +61 @@
     data->maskName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-mask"));
     data->outRoot = psStringCopy(argv[1]);
-    psMetadataAddStr(data->config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Output root name", data->outRoot);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "OUTPUT", 0, "Output root name", data->outRoot);
 
     psTrace("ppBackground", 1, "Done reading command-line arguments\n");
-    psFree(arguments);
 
     if (!data->patternName && !data->backgroundName) {
@@ -68 +69 @@
         return false;
     }
+    if (!data->imageName) {
+        psError(PPBACKGROUND_ERR_CONFIG, true, "Must specify -image");
+        return false;
+    }
 
     PS_ASSERT_STRING_NON_EMPTY(data->outRoot, false);
+
+    const char *statsName = psMetadataLookupStr(NULL, arguments, "-stats");
+    if (statsName) {
+        data->statsFile = fopen(statsName, "w");
+        if (!data->statsFile) {
+            psError(PPBACKGROUND_ERR_IO, true, "Unable to open statistics file %s", statsName);
+            return false;
+        }
+        data->stats = psMetadataAlloc();
+        pmConfigRunFilenameAddWrite(data->config, "STATS", statsName);
+    }
 
     return true;

branches/czw_branch/20100519/ppBackground/src/ppBackgroundCamera.c

@@ -32 +32 @@
     bool status;                        // Status of file definition
 
-    fileArguments("PATTERN", data->patternName, "Input pattern", data->config);
-    pmFPAfile *pattern = pmFPAfileDefineFromArgs(&status, data->config, "PPBACKGROUND.PATTERN",
-                                                 "PATTERN"); // File
-    if (!status) {
-        psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.PATTERN");
-        return false;
+    if (data->patternName) {
+        fileArguments("PATTERN", data->patternName, "Input pattern", data->config);
+        pmFPAfileDefineFromArgs(&status, data->config, "PPBACKGROUND.PATTERN", "PATTERN"); // File
+        if (!status) {
+            psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.PATTERN");
+            return false;
+        }
     }
 
-    fileArguments("BACKGROUND", data->backgroundName, "Input background model", data->config);
-    pmFPAfile *background = pmFPAfileBindFromArgs(&status, data->config, "PPBACKGROUND.BACKGROUND",
-                                                  "BACKGROUND"); // File
-    if (!status) {
-        psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.BACKGROUND");
-        return false;
+    if (data->backgroundName) {
+        fileArguments("BACKGROUND", data->backgroundName, "Input background model", data->config);
+        pmFPAfileDefineFromArgs(&status, data->config, "PPBACKGROUND.BACKGROUND", "BACKGROUND"); // File
+        if (!status) {
+            psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.BACKGROUND");
+            return false;
+        }
     }
 
-    fileArguments("IMAGE", data->backgroundName, "Input uncorrected image", data->config);
-    pmFPAfile *image = pmFPAfileBindFromArgs(&status, background, data->config, "PPBACKGROUND.IMAGE",
-                                             "IMAGE"); // File
-    if (!status) {
+    fileArguments("IMAGE", data->imageName, "Input uncorrected image", data->config);
+    pmFPAfile *image = pmFPAfileDefineFromArgs(&status, data->config, "PPBACKGROUND.IMAGE",
+                                               "IMAGE"); // File
+    if (!status || !image) {
         psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.IMAGE");
         return false;
     }
 
-    fileArguments("MASK", data->backgroundName, "Input uncorrected mask", data->config);
-    pmFPAfile *mask = pmFPAfileBindFromArgs(&status, image, data->config, "PPBACKGROUND.MASK",
-                                              "MASK"); // File
+    fileArguments("MASK", data->maskName, "Input uncorrected mask", data->config);
+    pmFPAfileBindFromArgs(&status, image, data->config, "PPBACKGROUND.MASK", "MASK"); // File
     if (!status) {
         psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.MASK");
@@ -64 +65 @@
     }
 
-    pmFPAfile *output = pmFPAfileDefineOutput(data->config, file->fpa, "PPBACKGROUND.OUTPUT");
+    pmFPAfile *output = pmFPAfileDefineOutput(data->config, image->fpa, "PPBACKGROUND.OUTPUT");
     if (!output) {
         psError(psErrorCodeLast(), false, "Unable to define output.");
@@ -70 +71 @@
     }
     output->save = true;
+
+    pmFPAfile *outMask = pmFPAfileDefineOutput(data->config, output->fpa, "PPBACKGROUND.OUTPUT.MASK");
+    if (!outMask) {
+        psError(psErrorCodeLast(), false, "Unable to define output.");
+        return false;
+    }
+    outMask->save = true;
 
 #if 0

branches/czw_branch/20100519/ppBackground/src/ppBackgroundExit.c

@@ -13 +13 @@
     if (errorCode != PS_ERR_NONE) {
         pmFPAfileFreeSetStrict(false);
-        psErrorStackPrint(stderr, "Error in subtraction:");
+        psErrorStackPrint(stderr, "Error in background restoration:");
         switch (errorCode) {
           case PS_ERR_UNKNOWN:

branches/czw_branch/20100519/ppBackground/src/ppBackgroundLoop.c

@@ -6 +6 @@
 #include <pslib.h>
 #include <psmodules.h>
+#include <ppStats.h>
 
 #include "ppBackground.h"
@@ -13 +14 @@
 {
     pmConfig *config = data->config;                                        // Configuration data
-    pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPBACKGROUND.BACKGROUND", 0);
+    pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPBACKGROUND.IMAGE", 0);
     pmFPAfile *patternFile = pmFPAfileSelectSingle(config->files, "PPBACKGROUND.PATTERN", 0); // File with data
+    pmFPAfile *bgFile = pmFPAfileSelectSingle(config->files, "PPBACKGROUND.BACKGROUND", 0);   // File with bg
     pmFPA *patternMosaic = pmFPAConstruct(file->camera, file->cameraName); // Mosaicked FPA for pattern
 
-    psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Mask value to set
+    psImageMaskType maskBad = pmConfigMaskGet("DETECTOR", config); // Mask value to set
 
     pmFPAview *view = pmFPAviewAlloc(0); // Pointer into FPA hierarchy
@@ -34 +36 @@
         }
 
-        if (chip->cell->n > 1) {
+        if (chip->cells->n > 1) {
             psError(PPBACKGROUND_ERR_CONFIG, true,
                     "Input image and background model should be chip-mosaicked");
@@ -44 +46 @@
             pmFPAfileActivate(config->files, true, "PPBACKGROUND.PATTERN");
 
-            pmCell *cell;               // Cell with pattern data
+            pmCell *patternCell;               // Cell with pattern data
             while ((patternCell = pmFPAviewNextCell(view, patternFile->fpa, 1))) {
-                if (!cell->process || !cell->file_exists) {
+                if (!patternCell->process || !patternCell->file_exists) {
                     continue;
                 }
@@ -68 +70 @@
                     // Get size of image from concepts
                     if (numCols == 0 || numRows == 0) {
-                        numCols = psMetadataLookupS32(NULL, cell->concepts, "CELL.XSIZE");
-                        numRows = psMetadataLookupS32(NULL, cell->concepts, "CELL.YSIZE");
+                        numCols = psMetadataLookupS32(NULL, patternCell->concepts, "CELL.XSIZE");
+                        numRows = psMetadataLookupS32(NULL, patternCell->concepts, "CELL.YSIZE");
                     }
                     // Get size of image from TRIMSEC
                     if (numCols == 0 || numRows == 0) {
-                        psRegion *trimsec = psMetadataLookupPtr(NULL, cell->concepts, "CELL.TRIMSEC");
+                        psRegion *trimsec = psMetadataLookupPtr(NULL, patternCell->concepts, "CELL.TRIMSEC");
                         numCols = trimsec->x1 - trimsec->x0;
                         numRows = trimsec->y1 - trimsec->y0;
@@ -84 +86 @@
                     readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32);
                     psImageInit(readout->image, 0.0);
+                    readout->mask = psImageAlloc(numCols, numRows, PS_TYPE_IMAGE_MASK);
+                    psImageInit(readout->mask, 0);
 
                     if (!pmPatternRowApply(readout, maskBad) || !pmPatternCellApply(readout, maskBad)) {
@@ -91 +95 @@
 
                     // Remove bias sections to avoid warnings
-                    psMetadataItem *biassec = psMetadataLookup(patternRO->parent->concepts, "CELL.BIASSEC");
+                    psMetadataItem *biassec = psMetadataLookup(patternCell->concepts, "CELL.BIASSEC");
                     if (psListLength(biassec->data.V)) {
                         psFree(biassec->data.V);
@@ -114 +118 @@
             }
 
-            pmChip *mosaic = pmFPAviewThisChip(view, patternMosaic); // Chip for mosaicked pattern
-            if (!pmChipMosaic(mosaic, chip, true, maskBad)) {
-                psError(psErrorCodeLast(), false, "Unable to mosaic pattern correction");
-                return false;
+            if (data->patternName) {
+                pmChip *patternChip = pmFPAviewThisChip(view, patternFile->fpa); // Chip for pattern
+                pmChip *mosaic = pmFPAviewThisChip(view, patternMosaic); // Chip for mosaicked pattern
+                if (!mosaic->hdu && !mosaic->parent->hdu) {
+                    pmFPAAddSourceFromView(patternMosaic, view, file->format);
+                }
+                if (!pmChipMosaic(mosaic, patternChip, true, maskBad)) {
+                    psError(psErrorCodeLast(), false, "Unable to mosaic pattern correction");
+                    return false;
+                }
+                pmFPAfileActivate(config->files, true, NULL);
+            //            pmFPAfileActivate(config->files, false, "PPBACKGROUND.PATTERN");
             }
-            pmFPAfileActivate(config->files, true, NULL);
-            pmFPAfileActivate(config->files, false, "PPBACKGROUND.PATTERN");
         }
 
+        pmChip *patternChip = patternFile ? pmFPAviewThisChip(view, patternMosaic) : NULL; // Chip with pattern
+        pmChip *bgChip = bgFile ? pmFPAviewThisChip(view, bgFile->fpa) : NULL; // Chip with background model
+        if (!ppBackgroundRestore(chip, bgChip, patternChip, view, config)) {
+            psError(psErrorCodeLast(), false, "Unable to replace background");
+            return false;
+        }
+        pmChipFreeData(patternChip);
 
-
-
-
-
-
-                pmHDU *hdu = pmHDUGetLowest(file->fpa, chip, cell); // HDU for readout
-                if (!hdu) {
-                    psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find HDU for data.");
-                    return false;
-                }
-                if (!hdu->header) {
-                    hdu->header = psMetadataAlloc();
-                }
-                ppBackgroundVersionHeader(hdu->header);
-
-
-
-
-
-
-
+        pmHDU *hdu = pmHDUGetLowest(file->fpa, chip, NULL); // HDU for chip
+        if (!hdu) {
+            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find HDU for data.");
+            return false;
+        }
+        if (!hdu->header) {
+            hdu->header = psMetadataAlloc();
+        }
+        ppBackgroundVersionHeader(hdu->header);
 
         // Chip
@@ -158 +163 @@
     }
 
+    psFree(patternMosaic);
+
+    if (data->stats) {
+        psImageMaskType maskVal = 0;    // Bits to mask
+        if (!pmConfigMaskSetBits(&maskVal, NULL, config)) {
+            psError(psErrorCodeLast(), false, "Unable to find bits to mask");
+            return false;
+        }
+        ppStatsFPA(data->stats, file->fpa, view, maskVal, config);
+    }
+
+    psFree(view);
+
     return true;
 }

branches/czw_branch/20100519/ppBackground/src/ppBackgroundVersionDefinitions.h.in

@@ -1 @@
-#ifndef PPVIZPATTERN_VERSION_DEFINITIONS_H
-#define PPVIZPATTERN_VERSION_DEFINITIONS_H
+#ifndef PPBACKGROUND_VERSION_DEFINITIONS_H
+#define PPBACKGROUND_VERSION_DEFINITIONS_H
 
-#define PPVIZPATTERN_VERSION @PPVIZPATTERN_VERSION@ // SVN version
-#define PPVIZPATTERN_BRANCH  @PPVIZPATTERN_BRANCH@  // SVN branch
-#define PPVIZPATTERN_SOURCE  @PPVIZPATTERN_SOURCE@  // SVN source
+#define PPBACKGROUND_VERSION @PPBACKGROUND_VERSION@ // SVN version
+#define PPBACKGROUND_BRANCH  @PPBACKGROUND_BRANCH@  // SVN branch
+#define PPBACKGROUND_SOURCE  @PPBACKGROUND_SOURCE@  // SVN source
 
 #endif

branches/czw_branch/20100519/ppImage/src/ppImageLoop.c

@@ -51 +51 @@
             ESCAPE("load failure for Chip");
         }
-        printf("PPI: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
         // crosstalk measurement needs to be done on the entire chip at once, and before
         // signal levels are modified by the detrending.  If crosstalk measurement is
@@ -65 +65 @@
           ESCAPE("Unable to perform crosstalk correction");
         }
-        printf("PPI: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
         psTimerStart(TIMER_DETREND);
         pmCell *cell;                   // Cell from chip
@@ -152 +152 @@
             ESCAPE("Unable to free detrend images");
         }
-        printf("PPI: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
         // Apply the fringe correction
         if (options->doFringe) {
@@ -163 +163 @@
             ESCAPE("Unable to free fringe images");
         }
-        printf("PPI: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
         // Apply the pattern correction
         if (options->doPatternRow || options->doPatternCell) {
@@ -170 +170 @@
           }
         }
-        printf("PPI: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
         // measure various pixel-based statistics for this image
         if (!ppImagePixelStats(config, stats, options, view)) {
             ESCAPE("Unable to measures pixel stats for image");
         }
-        printf("PPI: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
         if (!ppImageMosaicChip(config, options, view, "PPIMAGE.CHIP", "PPIMAGE.OUTPUT")) {
             ESCAPE("Unable to mosaic chip");
@@ -182 +182 @@
 
         // we perform photometry on the readouts of this chip in the output
-        printf("PPI: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
         psTimerStart(TIMER_PHOT);
         if (options->doPhotom) {

branches/czw_branch/20100519/ppImage/src/ppImageMosaic.c

@@ -9 +9 @@
 {
     bool status;                        // Status of MD lookup
-        printf("PPIM: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
     pmFPAfile *in = psMetadataLookupPtr(&status, config->files, inFile); // Input file
     if (!status) {
@@ -15 +15 @@
         exit(EXIT_FAILURE);
     }
-        printf("PPIM: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
     pmFPAfile *out = psMetadataLookupPtr(&status, config->files, outFile); // Output file
     if (!status) {
@@ -21 +21 @@
         exit(EXIT_FAILURE);
     }
-        printf("PPIM: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
     pmChip *outChip = pmFPAviewThisChip(view, out->fpa);
     pmChip *inChip = pmFPAviewThisChip(view, in->fpa);
@@ -27 +27 @@
         pmFPAAddSourceFromView(out->fpa, view, out->format);
     }
-        printf("PPIM: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
     // If the input is a dark (normalising and mosaicking) then it looks like a video cell and will be ignored
     // by pmChipMosaic.  We therefore hack the structure so it doesn't look like a video cell.
     psVector *darkNumbers = NULL;       // Number of dark readouts for each cell
-    //    psVectorInit(darkNumbers, 0);
-    printf("PPIM: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     if (psMetadataLookupBool(&status, config->arguments, "INPUT_IS_DARK")) {
         darkNumbers = psVectorAlloc(inChip->cells->n, PS_TYPE_S32);
@@ -45 +43 @@
         }
     }
-        printf("PPMI: %x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
+
     psTrace("pmChipMosaic", 5, "mosaic chip %s to %s (xbin,ybin: %d,%d to %d,%d)\n",
             in->name, out->name, in->xBin, in->yBin, out->xBin, out->yBin);

branches/czw_branch/20100519/ppStack/src/ppStackTarget.c

@@ -164 +164 @@
 
         psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); // Statistics for background
+        float mean = NAN;                                  // Measured mean variance
         if (!psImageBackground(bg, NULL, variance, mask, maskVal, rng)) {
-            psError(PPSTACK_ERR_DATA, false, "Can't measure background for image.");
-            psFree(rng);
-            return NULL;
+            psErrorClear();
+            // Retry using all the available pixels
+            bg->nSubsample = variance->numCols * variance->numRows + 1;
+            if (!psImageStats(bg, variance, mask, maskVal)) {
+                psLogMsg("ppStack", PS_LOG_DETAIL,
+                         "Couldn't measure mean variance for image %d; retrying.", i);
+                psErrorClear();
+                // Retry with desperate statistic
+                bg->options = PS_STAT_SAMPLE_MEAN;
+                if (!psImageStats(bg, variance, mask, maskVal)) {
+                    psWarning("Unable to measure mean variance for image %d --- rejecting.", i);
+                    psErrorStackPrint(stderr, "Unable to measure mean variance for image %d", i);
+                    options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PPSTACK_MASK_PSF;
+                    goto DONE;
+                } else {
+                    // Desperate retry
+                    mean = bg->sampleMean;
+                }
+            } else {
+                // Retry with all available pixels
+                mean = bg->robustMedian;
+            }
+        } else {
+            // First attempt
+            mean = bg->robustMedian;
         }
 
@@ -177 +200 @@
         }
 
+    DONE:
+        psFree(bg);
         if (!ppStackFilesIterateUp(config)) {
             psFree(rng);

branches/czw_branch/20100519/psLib/src/sys/psAbort.c

@@ -23 +23 @@
 #include <stdarg.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 
+#if defined(HAVE_BACKTRACE)
+#include <execinfo.h>
+#define BACKTRACE_BUFFER_SIZE 256       // Maximum size of backtrace
+#endif
+
 #include "psAbort.h"
+#include "psString.h"
+#include "psMemory.h"
 #include "psError.h"
 #include "psLogMsg.h"
+
+// Write backtrace to log
+static inline void psBacktrace(void)
+{
+#ifdef HAVE_BACKTRACE
+    void **bt = psAlloc(BACKTRACE_BUFFER_SIZE * sizeof(void *)); // Backtrace information
+    if (!bt) {
+        psLogMsg("psLib.sys", PS_LOG_ABORT, "Unable to allocate memory for backtrace");
+        return;
+    }
+    int size = backtrace(bt, BACKTRACE_BUFFER_SIZE);             // Size of backtrace
+    char **strings = backtrace_symbols((void *const *)bt, size);
+    psLogMsg("psLib.sys", PS_LOG_ABORT, "Backtrace depth: %d", size);
+    for (int i = 0; i < size; i++) {
+        // if the caller was an anon function then strchr won't
+        // find a '(' in the string and will return NULL
+        char *caller = strchr(strings[i], '(');
+        if (caller) {
+            // skip over the '('
+            caller++;
+            // find the end of the symbol name
+            size_t callerLength = abs(strchr(caller, '+') - caller);
+            psString name = psStringNCopy(caller, callerLength);
+            psLogMsg("psLib.sys", PS_LOG_ABORT, "Backtrace %d: %s", i, name);
+            psFree(name);
+        } else {
+            psLogMsg("psLib.sys", PS_LOG_ABORT, "Backtrace %d: (unknown)", i);
+        }
+    }
+#endif // ifdef HAVE_BACKTRACE
+}
 
 void p_psAbort(const char *file,
@@ -46 +85 @@
     // Clean up stack after variable arguement has been used
     va_end(argPtr);
+
+    psBacktrace();
 
     // Call system abort function to terminate program execution
@@ -72 +113 @@
     va_end(argPtr);
 
+    psBacktrace();
+
     // Call system abort function to terminate program execution
     fsync(psLogGetDestination());

branches/czw_branch/20100519/psLib/src/sys/psThread.c

@@ -210 +210 @@
         psThreadTask *task = psHashLookup(tasks, job->type); // Task to execute job
         psAssert(task, "Couldn't find thread task %s", job->type);
-        psAssert(job->args->n == task->nArgs, "invalid number of arguments to %s (%ld supplied, expected %d)", task->type, job->args->n, task->nArgs);
+        psAssert(job->args->n == task->nArgs,
+                 "invalid number of arguments to %s (%ld supplied, expected %d)",
+                 task->type, job->args->n, task->nArgs);
         bool status = task->function(job); // Status of executing task
 

branches/czw_branch/20100519/psModules/src/config/pmConfig.c

@@ -306 +306 @@
     *config = psMetadataConfigRead(NULL, &numBadLines, realName, true);
     if (numBadLines > 0) {
-        psError(PM_ERR_CONFIG, false, "%d bad lines in %s configuration file (%s)",
+        psError(PM_ERR_CONFIG, true, "%d bad lines in %s configuration file (%s)",
                 numBadLines, description, realName);
         psFree (realName);
@@ -1751 +1751 @@
     psMetadataItem *item = psMetadataLookup(camera, "FILERULES"); // Item with the file rule of interest
     if (!item) {
-        psError(PM_ERR_CONFIG, false, "Unable to find FILERULES in the camera configuration.");
+        psError(PM_ERR_CONFIG, true, "Unable to find FILERULES in the camera configuration.");
         return NULL;
     }
@@ -1771 +1771 @@
     }
 
-    return psMetadataLookupMetadata(&mdok, filerules, realname);
+    return psMetadataLookupMetadata(NULL, filerules, realname);
 }
 

branches/czw_branch/20100519/pstamp/src/pstamp.h

@@ -33 +33 @@
 #define PSTAMP_SELECT_BACKMDL   32
 #define PSTAMP_SELECT_INVERSE 1024
+#define PSTAMP_SELECT_UNCONV  2048
+#define PSTAMP_USE_IMFILE_ID 16384
 
-#define PSTAMP_WAIT_FOR_UPDATE 2048
+#define PSTAMP_NO_WAIT_FOR_UPDATE 32768
 
 #define PSTAMP_CENTER_IN_PIXELS 1

branches/czw_branch/20100519/tools/chip_outputs.pl

@@ -23 +23 @@
                              'PSF' => '.psf',     # Point-spread function
                              'LOG' => '.log',     # Processing log
+                             'PATTERN' => '.ptn', # Pattern subtraction
+                             'MODEL' => '.mdl.fits', # Background model
                          };