Index: /branches/eam_branches/ipp-20130509/PS-IPP-Config/lib/PS/IPP/Config.pm
===================================================================
--- /branches/eam_branches/ipp-20130509/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 35746)
+++ /branches/eam_branches/ipp-20130509/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 35747)
@@ -730,4 +730,49 @@
}
+sub file_rename {
+ my $self = shift;
+ my $old_name = shift;
+ my $name = shift;
+
+ my $scheme = file_scheme($old_name);
+ my $newscheme = file_scheme($name);
+ if ($scheme and $scheme eq 'neb') {
+ if (!$newscheme or $newscheme ne 'neb') {
+ carp "cannot rename nebulous file to non nebulous path";
+ return undef;
+ }
+ my $neb = $self->nebulous;
+ if ($neb->storage_object_exists($old_name)) {
+ eval {
+ $neb->move($old_name, $name);
+ };
+ if ($@) {
+ carp "nebulous move failed for $old_name";
+ return undef;
+ }
+ } else {
+ carp "$old_name not found";
+ return undef;
+ }
+ } else {
+ if ($newscheme and $newscheme eq 'neb') {
+ carp "cannot rename non-nebulous file to nebulous path";
+ return undef;
+ }
+ if ($self->file_exists($old_name)) {
+ my $resolved = $self->file_resolve($old_name);
+ my $resolved_new = $self->file_resolve($name);
+ if (!rename($resolved, $resolved_new)) {
+ carp "Failed to rename $resolved $resolved_new";
+ return undef;
+ }
+ } else {
+ carp "$old_name not found";
+ return undef;
+ }
+ }
+ return $name;
+}
+
# redirect stderr and stdout streams to log file. If file already exists rename it and record
# the name in the new log file.
Index: /branches/eam_branches/ipp-20130509/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm
===================================================================
--- /branches/eam_branches/ipp-20130509/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35746)
+++ /branches/eam_branches/ipp-20130509/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35747)
@@ -574,6 +574,6 @@
$command .= " -dateobs_end $dateobs_end" if $dateobs_end;
} elsif ($req_type ne "byid") {
- $command .= " -mjd_min $mjd_min" if $mjd_min;
- $command .= " -mjd_max $mjd_max" if $mjd_max;
+ $command .= " -mjd_obs_begin $mjd_min" if $mjd_min;
+ $command .= " -mjd_obs_end $mjd_max" if $mjd_max;
}
@@ -1025,16 +1025,16 @@
} else {
$using_camtool = 1;
- my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
- # NOTE: we are applying the data_group to the camera run.
- # If we're looking for chip stage images there is no guarentee that
- # chipRun.data_group eq camRun.data_group. In practice this is almost
- # always the case. If this turns out to be a problem we can defer
- # the data_group test to when we look up the chipProcessedImfiles
- if ($data_group) {
- $command .= " -data_group $data_group";
- } else {
- $command .= $default_data_groups;
- }
- $command .= " -destreaked" if $need_magic;
+ $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
+ # NOTE: we are applying the data_group to the camera run.
+ # If we're looking for chip stage images there is no guarentee that
+ # chipRun.data_group eq camRun.data_group. In practice this is almost
+ # always the case. If this turns out to be a problem we can defer
+ # the data_group test to when we look up the chipProcessedImfiles
+ if ($data_group) {
+ $command .= " -data_group $data_group";
+ } else {
+ $command .= $default_data_groups;
+ }
+ $command .= " -destreaked" if $need_magic;
}
$command .= " -ra $ra -decl $dec -radius $search_radius";
Index: /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/Nebulous.pm
===================================================================
--- /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/Nebulous.pm (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/Nebulous.pm (revision 35747)
@@ -113,5 +113,5 @@
} elsif (($line =~ m/(ipp[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)||
($line =~ m/(ippb[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) ||
- ($line =~ m/(stsci0[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
+ ($line =~ m/(stsci[0-1][0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
$self->{_totalHosts}++;
Index: /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/Plotter.pm
===================================================================
--- /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/Plotter.pm (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/Plotter.pm (revision 35747)
@@ -707,5 +707,5 @@
if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
print GP
- "set term $self->{_outputFormat} size 1280,1024;" .
+ "set term $self->{_outputFormat} size 2560,1024;" .
"set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" .
"set style fill solid 1.00 border -1;" .
Index: /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/czarconfig.xml
===================================================================
--- /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/czarconfig.xml (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippMonitor/czartool/czartool/czarconfig.xml (revision 35747)
@@ -47,5 +47,5 @@
20 MINUTE
- stdscience distribution summitcopy registration pstamp stack publishing
+ stdscience distribution summitcopy registration stack publishing pstamp
Index: /branches/eam_branches/ipp-20130509/ippMonitor/czartool/sql/insert_new_stsci_nodes_stsciwave2.sql
===================================================================
--- /branches/eam_branches/ipp-20130509/ippMonitor/czartool/sql/insert_new_stsci_nodes_stsciwave2.sql (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippMonitor/czartool/sql/insert_new_stsci_nodes_stsciwave2.sql (revision 35747)
@@ -0,0 +1,30 @@
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci10.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci10.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci10.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci11.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci11.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci11.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci12.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci12.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci12.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci13.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci13.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci13.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci14.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci14.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci14.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci15.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci15.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci15.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci16.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci16.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci16.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci17.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci17.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci17.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci18.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci18.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci18.2', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci19.0', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci19.1', 35016743644, 0, 0, 0, 0);
+INSERT INTO czardb.hosts(host, total, available, used, writable, readable) VALUES('stsci19.2', 35016743644, 0, 0, 0, 0);
Index: /branches/eam_branches/ipp-20130509/ippMonitor/def/diffProcessedSkyfile_Images.d
===================================================================
--- /branches/eam_branches/ipp-20130509/ippMonitor/def/diffProcessedSkyfile_Images.d (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippMonitor/def/diffProcessedSkyfile_Images.d (revision 35747)
@@ -3,4 +3,7 @@
FILE diffProcessedSkyfile_Images.php
MENU ipp.stack.dat
+
+UNRESTRICTED AND 0>1
+RESTRICT_COUNT AND 0>1
WHERE diffRun.diff_id = diffSkyfile.diff_id
@@ -13,5 +16,5 @@
# if no query restrictions are supplied, we want to limit the query or it is extremely long running:
-UNRESTRICTED AND diffRun.diff_id = 0
+# UNRESTRICTED AND diffRun.diff_id = 0
# XXX need to get camera from lookup
Index: /branches/eam_branches/ipp-20130509/ippMonitor/def/newsimplePlotStaticskyImage.d
===================================================================
--- /branches/eam_branches/ipp-20130509/ippMonitor/def/newsimplePlotStaticskyImage.d (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippMonitor/def/newsimplePlotStaticskyImage.d (revision 35747)
@@ -0,0 +1,26 @@
+TABLE staticskyRun, staticskyInput, stackRun, skycell
+TITLE New sky plot - staticsky
+FILE simplePlotStaticskyImage.php
+MENU ipp.plots.dat
+
+WHERE staticskyRun.state != 'new'
+WHERE staticskyRun.sky_id = staticskyInput.sky_id
+WHERE staticskyInput.stack_id = stackRun.stack_id
+WHERE stackRun.skycell_id = skycell.skycell_id
+WHERE stackRun.tess_id = skycell.tess_id
+
+# field size format name show link to extras
+FIELD staticskyRun.sky_id, 5, %d, Staticsky ID, restrict
+FIELD stackRun.stack_id, 5, %d, Stack ID, restrict
+FIELD staticskyRun.label, 10, %s, Label, restrict
+FIELD staticskyRun.state, 10, %s, state, restrict
+FIELD skycell.radeg, 10, %.6f, RA (deg), restrict
+FIELD skycell.decdeg, 10, %.6f, DEC (deg), restrict
+FIELD stackRun.filter, 10, %s, FILTER, restrict
+#FIELD staticskyResult.num_inputs, 5, %d, num_inputs, restrict
+#FIELD staticskyResult.quality, 5, %d, quality, restrict
+
+# What to plot
+TOPLOT skycell.radeg,skycell.decdeg,stackRun.filter
+PLOTTER skycellplot.php
+PLOTTITLE Sky Plot Staticsky
Index: /branches/eam_branches/ipp-20130509/ippScripts/scripts/background_chip.pl
===================================================================
--- /branches/eam_branches/ipp-20130509/ippScripts/scripts/background_chip.pl (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippScripts/scripts/background_chip.pl (revision 35747)
@@ -159,7 +159,4 @@
}
-# XXX: M31 TEST turn off restoring pattern correction
-$apply_pattern = 0;
-
# Set up files
&my_die("Couldn't find input file: $in_image\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_image);
@@ -170,9 +167,4 @@
$ipprc->outroot_prepare($outroot);
-#my $out_image = $ipprc->filename("PPBACKGROUND.OUTPUT", $outroot, $class_id);
-#my $out_mask = $ipprc->filename("PPBACKGROUND.OUTPUT.MASK", $outroot, $class_id);
-#my $out_stats = $ipprc->filename("PPBACKGROUND.STATS", $outroot, $class_id);
-#my $out_config = $ipprc->filename("PPBACKGROUND.CONFIG", $outroot, $class_id);
-#my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id);
my $out_image = prepare_output("PPBACKGROUND.OUTPUT", $outroot, $class_id, 1);
@@ -183,74 +175,6 @@
my $traceDest = prepare_output("TRACE.IMFILE", $outroot, $class_id, 1);
-my $auxmask;
-
-my $apply_auxiliary_mask = 1;
-# XXX Temporary lookup of auxiliary mask file from static location.
-# At some point I'll convert this to register the masks in the database and use detselect
-if ($apply_auxiliary_mask) {
- my @auxmask_start_date = qw(
- 0000
- 5178
- 5218
- 5403
- 5424
- 5455
- 5485
- 5512
- 5545
- 5770
- 5797
- 5825
- 5854
- 5881
- 6138
- 6153
- 6197
- 6214
- );
- my @auxmask_end_date = qw(
- 5177
- 5217
- 5402
- 5423
- 5454
- 5484
- 5511
- 5544
- 5769
- 5796
- 5824
- 5853
- 5880
- 6137
- 6152
- 6196
- 6213
- 9999
- );
-
- my $outroot_base = basename($outroot);
-
- # Assume that outroot base begins with exp_name: fix by passing in exp_name
- my $mjd = substr($outroot_base, 1, 4);
- if ($mjd =~ /\D/) {
- my $error_code = 1;
- &my_die("unexpected outroot base: $outroot_base: $error_code", $chip_bg_id, $class_id, $error_code);
- }
-
- my $i = 0;
- while ($auxmask_end_date[$i] < $mjd) {
- $i++;
- }
-
- my $auxmask_base = "/data/ipp064.0/bills/m31-masks/www.usm.uni-muenchen.de/people/arri/PAndromeda/masks_fits/mask";
-
- $auxmask = sprintf "%s_%04d-%04d_%s.fits", $auxmask_base, $auxmask_start_date[$i], $auxmask_end_date[$i], $class_id;
- print "Auxililary mask file is $auxmask\n";
-}
-
-
-
-# Run ppImage
+
+# Run ppBackground
unless ($no_op) {
my $command = "$ppBackground $outroot";
@@ -258,5 +182,4 @@
$command .= " -mask $in_mask";
$command .= " -variance $in_wt";
- $command .= " -auxmask $auxmask" if $auxmask;
$command .= " -stats $out_stats" if $do_stats;
$command .= " -background $in_bg" if $apply_bg;
Index: /branches/eam_branches/ipp-20130509/ippScripts/scripts/background_warp.pl
===================================================================
--- /branches/eam_branches/ipp-20130509/ippScripts/scripts/background_warp.pl (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippScripts/scripts/background_warp.pl (revision 35747)
@@ -191,4 +191,6 @@
$command .= " $outroot $skyFile";
$command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
+ # XXX: change fits type based on recipe
+ $command .= " -R PSWARP.OUTPUT FITS.TYPE NONE";
$command .= " -recipe PSWARP $recipe_pswarp";
$command .= " -recipe PPSTATS WARPSTATS";
Index: /branches/eam_branches/ipp-20130509/ippScripts/scripts/camera_exp.pl
===================================================================
--- /branches/eam_branches/ipp-20130509/ippScripts/scripts/camera_exp.pl (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippScripts/scripts/camera_exp.pl (revision 35747)
@@ -53,5 +53,5 @@
'skip-binned' => \$skip_binned, # override recipe - don't create binned images
'skip-refmask' => \$skip_masks, # override recipe - don't create refmask
- 'bkg-only' => \$bkg_only, # override recipe - only do background continuity
+# 'bkg-only' => \$bkg_only, # override recipe - only do background continuity
'verbose' => \$verbose, # Print to stdout
'no-update' => \$no_update, # Update the database?
@@ -80,4 +80,6 @@
my $replicateOutputs = 1;
+$ipprc->outroot_prepare($outroot);
+
my $logDest;
my $traceDest;
@@ -114,4 +116,18 @@
my $bkg_recipe = 'PPIMAGE_BKGCONT'; # Add to reduction?
my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $recipeData;
+{
+ # Get the PSASTRO recipe
+ my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
+ }
+ $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+ &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
+}
my $cmdflags;
@@ -158,53 +174,22 @@
chomp $cmdflags;
- { # Determine if FWHM is too large to bother continuing.
- my $maxFWHM = 0;
- # XXX: temporary hack if this is a CNP exposure set maxFWHM to a large
- # value.
- # Before the next tag we will put this into a recipe
- if ($outroot =~ /CNP/) {
- $maxFWHM = 25;
- } else {
- my $command = "$ppConfigDump -camera $camera -get-key MAX_ALLOWED_FWHM";
- my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => $verbose);
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
- }
- if (@$stdout_buf) {
- my $cameraConfig = $mdcParser->parse(join "", @$stdout_buf) or
- &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
- $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM');
- }
- }
- if ($maxFWHM) {
- my $expFWHM;
- ($expFWHM) = $cmdflags =~ /-fwhm_major (\d+)/;
-
- if ($expFWHM > $maxFWHM) {
- print "Setting quality to 4007 due to large FWHM: exposure: $expFWHM maximum: $maxFWHM\n";
- $cmdflags .= " -quality 4007 "; # This corresponds to PSASTRO_ERR_DATA
- $no_op = 1;
- }
- }
- }
-}
-
-
-my $produceMasks; # Produce masks?
+ # Determine if FWHM is too large to bother continuing.
+ my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM');
+ if ($maxFWHM) {
+ my $expFWHM;
+ ($expFWHM) = $cmdflags =~ /-fwhm_major (\d+)/;
+
+ if ($expFWHM > $maxFWHM) {
+ print "Setting quality to 4007 due to large FWHM: exposure: $expFWHM maximum: $maxFWHM\n";
+ $cmdflags .= " -quality 4007 "; # This corresponds to PSASTRO_ERR_DATA
+ $no_op = 1;
+ }
+ }
+}
+
+
+my $do_masks; # Produce masks?
if (!$skip_masks) {
- # Get the PSASTRO recipe
- my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => $verbose);
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
- }
- my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
- &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
-
- $produceMasks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
+ $do_masks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
}
@@ -220,9 +205,65 @@
my ($list5File, $list5Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b5.list.XXXX", UNLINK => !$save_temps ); # For background models
-# XXX we perform astrometry iff photometry output exists
+
+my $do_stats;
+my $do_bkg;
+my $do_jpegs;
+my $fpaStats;
+my $psastroInputArg;
+if ($run_state eq 'new') {
+ $do_stats = 1;
+ $do_bkg = 1;
+ $do_jpegs = !$skip_binned;
+ $fpaStats = prepare_output("PSASTRO.STATS", $outroot, undef, 1);
+ $psastroInputArg = " -list $list3Name";
+} else {
+ # for $run_state eq 'update' we only rebuild the masks using psastro
+ $do_stats = 0;
+ $do_jpegs = 0;
+ $do_bkg = 1; # we could skip this step if camProcessedExp.backgroun_model is non zero
+
+ if (!$do_masks) {
+ &my_die("run_state is update but do_masks is F. I have nothing to do!!", $cam_id, $PS_EXIT_UNKNOWN_ERROR)
+ }
+
+ # the input to psastro is the original PSASTRO.OUTPUT
+
+ # Check for file with rule PSASTRO.OUTPUT.ORGINAL
+ # This file will exist if we attempted to update this camRun before but faulted
+ # XXX: make sure cleanup deals with these files
+ my $inputObjects = $ipprc->filename("PSASTRO.OUTPUT.ORIGINAL", $outroot, undef);
+ if (!$ipprc->file_exists($inputObjects)) {
+ # not found so original file should still be in place.
+ # Rename it.
+ my $originalObjects = $ipprc->filename("PSASTRO.OUTPUT", $outroot, undef);
+ if ($ipprc->file_exists($originalObjects)) {
+
+ unless ($ipprc->file_rename($originalObjects, $inputObjects)) {
+ &my_die("failed to rename $originalObjects to $inputObjects", $cam_id, $PS_EXIT_UNKNOWN_ERROR);
+ }
+
+ # ok ready to go
+ } else {
+ print STDERR "failed to find input objects to update\n";
+ print STDERR "Original file name: $originalObjects\n";
+ print STDERR "Saved file name: $inputObjects\n";
+
+ &my_die("Cannot proceed.", $cam_id, $PS_EXIT_DATA_ERROR);
+
+ # XXX: actually we could use the chip stage cmfs as inputs and turn on astrometry
+ # ... except that somebody (me) got the bright idea to save space and
+ # changed chip stage cleanup to remove them. So we're kind of stuck.
+ }
+ }
+ $psastroInputArg = " -file $inputObjects -skipastro";
+}
+
+
my @outMasks; # Names of output masks
my @bkg_models; # Names of output background models
foreach my $file (@$files) {
+ # we perform astrometry iff photometry output exists
next if $file->{quality} != 0;
+
# use the path_base as OUTPUT root and convert the filenames with ipprc->filename:
my $class_id = $file->{class_id};
@@ -239,6 +280,6 @@
print $list5File ($ipprc->filename("PSPHOT.BACKMDL", $file->{path_base}, $class_id) . "\n");
- push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $produceMasks;
- push @bkg_models, prepare_output("PPIMAGE.BACKMDL", $outroot, $class_id, 1);
+ push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $do_masks;
+ push @bkg_models, prepare_output("PPIMAGE.BACKMDL", $outroot, $class_id, 1) if $do_bkg;
}
close $list1File;
@@ -247,21 +288,21 @@
close $list4File;
close $list5File;
-# Output products
-$ipprc->outroot_prepare($outroot);
+
+# Prepare the Output products
# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs
-my $jpeg1 = prepare_output("PPIMAGE.JPEG1", $outroot, undef, 1);
-my $jpeg2 = prepare_output("PPIMAGE.JPEG2", $outroot, undef, 1);
-my $fpaObjects = prepare_output("PSASTRO.OUTPUT", $outroot, undef, 1);
-my $configuration = prepare_output("PSASTRO.CONFIG", $outroot, undef, 1);
-
-my $do_stats;
-my $fpaStats;
+my $jpeg1 = prepare_output("PPIMAGE.JPEG1", $outroot, undef, 1) if $do_jpegs;
+my $jpeg2 = prepare_output("PPIMAGE.JPEG2", $outroot, undef, 1) if $do_jpegs;
+my $fpaObjects = prepare_output("PSASTRO.OUTPUT", $outroot, undef, 1) if !$bkg_only;
+
+my $configuration;
if ($run_state eq 'new') {
- $do_stats = 1;
- $fpaStats = prepare_output("PSASTRO.STATS", $outroot, undef, 1);
+ $configuration = prepare_output("PSASTRO.CONFIG", $outroot, undef, 1);
} else {
- $do_stats = 0;
-}
+ # Do not use the original recipes for updates because they might contain the
+ # recipe values that caused the masks to get fouled up in the first place
+ # $configuration = $ipprc->filename("PSASTRO.CONFIG", $outroot, undef);
+}
+
unless ($no_op) {
@@ -269,6 +310,6 @@
## build the output JPEG images first so we get them even if the astrometry fails
- # Make the jpeg for binning 1
- if (!$skip_binned and ($run_state eq 'new')) {
+ if ($do_jpegs) {
+ # Make the jpeg for binning 1
my $command = "$ppImage -list $list1Name $outroot"; # Command to run
$command .= " -recipe PPIMAGE $recipe1";
@@ -282,13 +323,12 @@
}
check_output($jpeg1, $replicateOutputs);
- }
-
- # Make the jpeg for binning 2
- if (!$skip_binned and ($run_state eq 'new')) {
- my $command = "$ppImage -list $list2Name $outroot"; # Command to run
+
+ # Make the jpeg for binning 2
+
+ $command = "$ppImage -list $list2Name $outroot"; # Command to run
$command .= " -recipe PPIMAGE $recipe2";
$command .= " -dbname $dbname" if defined $dbname;
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
run(command => $command, verbose => $verbose);
unless ($success) {
@@ -302,10 +342,10 @@
# run psastro on the chipObjects, producing fpaObjects
my $command;
- $command = "$psastro";
- $command .= " -list $list3Name";
- $command .= " -masklist $list4Name" if $produceMasks;
+ $command = $psastro;
+ $command .= " $psastroInputArg";
+ $command .= " -masklist $list4Name" if $do_masks;
$command .= " $outroot";
$command .= " -recipe PSASTRO $recipe_psastro";
- $command .= " -Db PSASTRO:REFSTAR_MASK F" if $skip_masks;
+ $command .= " -Db PSASTRO:REFSTAR_MASK F" if !$do_masks;
$command .= " -tracedest $traceDest -log $logDest";
$command .= " -dbname $dbname" if defined $dbname;
@@ -314,5 +354,5 @@
$command .= " -dumpconfig $configuration";
} elsif ($run_state eq 'update') {
- $command .= " -ipprc $configuration";
+ $command .= " -ipprc $configuration" if $configuration;
} else {
&my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR);
@@ -362,8 +402,9 @@
}
}
+
# Construct FPA continuity corrected background images
if ($camera =~ /ISP/) {
print "Skipping FPA continuity corrected background images for ISP\n";
- } else {
+ } elsif ($do_bkg) {
my $command;
$command = "$ppImage";
@@ -381,5 +422,16 @@
check_output($bkgModel, $replicateOutputs);
}
- $cmdflags = $cmdflags . " -background_model 1 ";
+ if ($run_state eq 'new') {
+ $cmdflags .= " -background_model 1 ";
+ } else {
+ my $command = "camtool -updateprocessedexp -set_background_model 1 -fault 0 -cam_id $cam_id";
+ $command .= " -dbname $dbname" if $dbname;
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to updateprocessedexp to set background_model: $error_code", $cam_id, $error_code);
+ }
+ }
}
}
Index: /branches/eam_branches/ipp-20130509/ippScripts/scripts/chip_imfile.pl
===================================================================
--- /branches/eam_branches/ipp-20130509/ippScripts/scripts/chip_imfile.pl (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippScripts/scripts/chip_imfile.pl (revision 35747)
@@ -200,5 +200,12 @@
## get the ppImage recipe for this camera and CHIP reduction
- $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe_ppImage -";
+ $command = "$ppConfigDump -dump-recipe PPIMAGE -recipe PPIMAGE $recipe_ppImage -";
+ if ($dump_config) {
+ # use this camera's recipes
+ $command .= " -camera $camera";
+ } else {
+ # get recipes as set in config dump file
+ $command .= " -ipprc $configuration";
+ }
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
run(command => $command, verbose => 0);
@@ -224,7 +231,9 @@
}
-
- ## XXX make the feature more general?
+ my $no_compress_image = metadataLookupBool($recipeData, 'NO.COMPRESS');
+
my $useDeburnedImage = metadataLookupBool($recipeData, 'USE.DEBURNED.IMAGE');
+ my $ppImageApplyBurntool = metadataLookupBool($recipeData, 'APPLY.BURNTOOL');
+ my $burntoolArguments;
if ($useDeburnedImage) {
@@ -327,134 +336,123 @@
}
- ## We now know that we have an image that has been burntooled.
- my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.deburned.XXXX",
+ # if recipe option PPIMAGE:APPLY.BURNTOOL is set and the burntool information is stored in
+ # an external table (usual case), check the existence of that table file and set $burntoolArguments.
+ # If burntool state does not match, say if the burntool data is stored in a fits extension in
+ # the rawImfile, use the burntool program.
+ if ($ppImageApplyBurntool && ($burntoolState == -1 * $burntoolStateGood)) {
+ my $burntoolTable_uri = $uri;
+ $burntoolTable_uri =~ s/fits$/burn.tbl/;
+ my $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+ if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
+ my $repair_cmd = "$nebrepair $burntoolTable_uri";
+ my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf )
+ = run(command => $repair_cmd, verbose => $verbose);
+ unless ($repair_success) {
+ &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id,
+ $PS_EXIT_SYS_ERROR);
+ }
+ $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+ }
+ unless ($ipprc->file_exists($burntoolTable_uri)) {
+ &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id,
+ $PS_EXIT_SYS_ERROR);
+ }
+ $burntoolArguments = " -burntool $burntoolTable_uri";
+ } else {
+ ## use external burntool program
+
+ ## We now know that we have an image that has been burntooled.
+ my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.deburned.XXXX",
UNLINK => !$save_temps, SUFFIX => '.fits' );
- # get the UNIX version of the (possible) neb: or path: filename
- my $uriReal = $ipprc->file_resolve( $uri );
- # Catch errors here
- if (!$uriReal) {
- my $repair_cmd = "$nebrepair $uri";
- my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
- unless ($repair_success) {
- &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
- $uriReal = $ipprc->file_resolve( $uri );
- }
-
- &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
-
- # funpack into the temp file.
- my $funpack_cmd = "$funpack -S $uriReal > $tempName";
- ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
- unless ($success) {
- # Catch errors here
- my $repair_cmd = "$nebrepair $uri";
- my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
- unless ($repair_success) {
- &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
- ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
- }
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
-
- ## Construct commands to apply the pre-calculated burntool trailfits to the pixel data.
- my ($burntoolTable_uri, $burntoolTable_uriReal);
- my $burntool_cmd = "$burntool ";
-
- if ($burntoolState == -1 * $burntoolStateGood) { # Burntool information stored in an external table.
- $burntoolTable_uri = $uri;
- $burntoolTable_uri =~ s/fits$/burn.tbl/;
- $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
- if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
- my $repair_cmd = "$nebrepair $burntoolTable_uri";
- my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
- unless ($repair_success) {
- &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
- $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
- }
- unless ($ipprc->file_exists($burntoolTable_uri)) {
- # Catch errors here
- &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR);
- }
-
- $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} persist=t apply=t";
- }
- elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table.
- $burntool_cmd .= "$tempName persist=t apply=t";
- }
- else {
- &my_die("Image data not properly burntooled, impossible state", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
-
- ## Run burntool to change the pixels of tempfile, and repoint $uri to that file.
- ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
- run(command => $burntool_cmd, verbose => $verbose);
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform burntool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
-
- $uri = $tempName;
- unless ($ipprc->file_exists($uri)) {
- &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
-
-# print STDERR "$uri $uriReal $tempName $burntoolTable_uri $burntoolTable_uriReal $burntool_cmd $save_temps\n";
-# exit(100);
- }
-
- # apply the 'tiltystreak' operation, if desired
+ # get the UNIX version of the (possible) neb: or path: filename
+ my $uriReal = $ipprc->file_resolve( $uri );
+ # Catch errors here
+ if (!$uriReal) {
+ my $repair_cmd = "$nebrepair $uri";
+ my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+ unless ($repair_success) {
+ &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+ }
+ $uriReal = $ipprc->file_resolve( $uri );
+ }
+
+ &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
+
+ # funpack into the temp file.
+ my $funpack_cmd = "$funpack -S $uriReal > $tempName";
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
+ unless ($success) {
+ # Catch errors here
+ my $repair_cmd = "$nebrepair $uri";
+ my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+ unless ($repair_success) {
+ &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+ }
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
+ }
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+ }
+
+ ## Construct commands to apply the pre-calculated burntool trailfits to the pixel data.
+ my ($burntoolTable_uri, $burntoolTable_uriReal);
+ my $burntool_cmd = "$burntool ";
+
+ if ($burntoolState == -1 * $burntoolStateGood) { # Burntool information stored in an external table.
+ $burntoolTable_uri = $uri;
+ $burntoolTable_uri =~ s/fits$/burn.tbl/;
+ $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+ if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
+ my $repair_cmd = "$nebrepair $burntoolTable_uri";
+ my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+ unless ($repair_success) {
+ &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+ }
+ $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+ }
+ unless ($ipprc->file_exists($burntoolTable_uri)) {
+ # Catch errors here
+ &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR);
+ }
+
+ $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} persist=t apply=t";
+ }
+ elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table.
+ $burntool_cmd .= "$tempName persist=t apply=t";
+ }
+ else {
+ &my_die("Image data not properly burntooled, impossible state", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+ }
+
+ ## Run burntool to change the pixels of tempfile, and repoint $uri to that file.
+ ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
+ run(command => $burntool_cmd, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform burntool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+ }
+
+ $uri = $tempName;
+ unless ($ipprc->file_exists($uri)) {
+ &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+ }
+
+ if ($dump_config) {
+ # tell ppImage to not apply burntool since we have done it here
+ $burntoolArguments = ' -Db PPIMAGE:APPLY.BURNTOOL F';
+ }
+ # print STDERR "$uri $uriReal $tempName $burntoolTable_uri $burntoolTable_uriReal $burntool_cmd $save_temps\n";
+ # exit(100);
+
+ }
+ }
+
my $tiltystreakApply = metadataLookupBool($recipeData, 'TILTYSTREAK.APPLY');
if ($tiltystreakApply) {
-
- my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS');
- my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id");
- if ($tiltystreakClassApply) {
-
- my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id");
- if ($tiltystreakClassOptions eq "none") {
- $tiltystreakClassOptions = "";
- }
-
- my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
-
- # XXX make these optional (must be built by psbuild as well...)
- my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-
- # create an temporary output file:
- my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps );
-
- print "uri: $uri\n";
-
- # get the UNIX version of the (possible) neb: or path: filename
- my $uriReal = $ipprc->file_resolve( $uri );
-
- print "uriReal: $uriReal\n";
-
- # unpack the data (is a NOP if not compressed)
- $command = "$funpack -S $uriReal > $tempName";
- ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
-
- # run tiltystreak
- $command = "$tiltystreak $tempName $tiltystreakClassOptions";
- ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
-
- # supply the output file as the new input file
- $uri = $tempName;
- }
+ # NOTE: application of TILTYSTREAK.APPLY was here. Removed after r35685
+ &my_die("Unable to perform tiltystreak: code has been removed", $exp_id, $chip_id, $class_id,
+ $PS_EXIT_PROG_ERROR);
}
@@ -479,4 +477,8 @@
$command .= " -Db PPIMAGE:BIN1.FITS FALSE -Db PPIMAGE:BIN2.FITS FALSE";
}
+ if ($no_compress_image) {
+ $command .= " -R PPIMAGE.CHIP FITS.TYPE NONE";
+ }
+ $command .= $burntoolArguments if $burntoolArguments;
$command .= " -threads $threads" if defined $threads;
$command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
Index: /branches/eam_branches/ipp-20130509/ippScripts/scripts/warp_skycell.pl
===================================================================
--- /branches/eam_branches/ipp-20130509/ippScripts/scripts/warp_skycell.pl (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippScripts/scripts/warp_skycell.pl (revision 35747)
@@ -129,4 +129,5 @@
my $astromSource; # The astrometry source
my $doBackground; # Do we want to make background models?
+my $noCompression;
{
my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -";
@@ -141,4 +142,5 @@
$astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
$doBackground = metadataLookupBool($metadata, 'BACKGROUND.MODEL');
+ $noCompression = metadataLookupBool($metadata, 'NO.COMPRESS');
}
@@ -299,4 +301,5 @@
$command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL";
$command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID";
+ $command .= " -R PSWARP.OUTPUT FITS.TYPE NONE" if $noCompression;
$command .= " -recipe PSWARP $recipe_pswarp";
$command .= " -tracedest $traceDest -log $logDest";
Index: /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_advancechip.sql
===================================================================
--- /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_advancechip.sql (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_advancechip.sql (revision 35747)
@@ -3,11 +3,19 @@
chipBackgroundImfile.magicked
FROM chipBackgroundRun
-JOIN chipProcessedImfile USING(chip_id)
-LEFT JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
+JOIN camRun using(cam_id)
+JOIN chipProcessedImfile ON chipBackgroundRun.chip_id = chipProcessedImfile.chip_id
+JOIN chipRun AS bgsub_chipRun on camRun.chip_id = bgsub_chipRun.chip_id
+JOIN chipProcessedImfile AS bgsub_chipProcessedImfile
+ ON (bgsub_chipRun.chip_id = bgsub_chipProcessedImfile.chip_id
+ AND bgsub_chipProcessedImfile.class_id = chipProcessedImfile.class_id)
+LEFT JOIN chipBackgroundImfile
+ ON (chipBackgroundRun.chip_bg_id = chipBackgroundImfile.chip_bg_id
+ AND chipBackgroundImfile.class_id = chipProcessedImfile.class_id)
WHERE chipBackgroundRun.state = 'new'
AND chipProcessedImfile.quality = 0
AND chipProcessedImfile.fault = 0
+ AND bgsub_chipProcessedImfile.quality = 0
-- WHERE hook %s
-GROUP BY chip_bg_id
+GROUP BY chipBackgroundRun.chip_bg_id
HAVING
COUNT(chipBackgroundImfile.class_id) = COUNT(chipProcessedImfile.class_id)
Index: /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_definewarp.sql
===================================================================
--- /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_definewarp.sql (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_definewarp.sql (revision 35747)
@@ -15,5 +15,5 @@
JOIN chipBackgroundRun ON chipRun.chip_id = chipBackgroundRun.chip_id
AND (chipBackgroundRun.cam_id = camRun.cam_id OR chipBackgroundRun.cam_id = 0)
-LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.warp_bg_id -- label hook %s
+LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.chip_bg_id -- label hook %s
WHERE chipBackgroundRun.state = 'full'
AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- need warp to have completed so warpSkyCellMap is populated
Index: /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_tochip.sql
===================================================================
--- /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_tochip.sql (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippTools/share/bgtool_tochip.sql (revision 35747)
@@ -1,5 +1,5 @@
SELECT
chipBackgroundRun.*,
- class_id,
+ chipProcessedImfile.class_id,
rawExp.exp_tag,
rawExp.camera
@@ -8,5 +8,12 @@
JOIN rawExp USING(exp_id)
JOIN chipProcessedImfile USING(chip_id)
-LEFT JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
+JOIN camRun using(cam_id)
+JOIN chipRun AS bgsub_chipRun on camRun.chip_id = bgsub_chipRun.chip_id
+JOIN chipProcessedImfile AS bgsub_chipProcessedImfile
+ ON (bgsub_chipRun.chip_id = bgsub_chipProcessedImfile.chip_id
+ AND bgsub_chipProcessedImfile.class_id = chipProcessedImfile.class_id)
+LEFT JOIN chipBackgroundImfile
+ ON (chipBackgroundRun.chip_bg_id = chipBackgroundImfile.chip_bg_id
+ AND chipBackgroundImfile.class_id = chipProcessedImfile.class_id)
LEFT JOIN Label ON chipBackgroundRun.label = Label.label
WHERE chipBackgroundImfile.chip_bg_id IS NULL
@@ -14,4 +21,5 @@
AND chipProcessedImfile.fault = 0
AND chipProcessedImfile.quality = 0
+ AND bgsub_chipProcessedImfile.quality = 0
AND (Label.active OR Label.active IS NULL)
-- WHERE hook %s
Index: /branches/eam_branches/ipp-20130509/ippTools/src/bgtool.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ippTools/src/bgtool.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippTools/src/bgtool.c (revision 35747)
@@ -774,6 +774,6 @@
psMetadata *where = psMetadataAlloc();
- PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "==");
- pxAddLabelSearchArgs(config, where, "-label", "label", "==");
+ PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+ pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "==");
PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
Index: /branches/eam_branches/ipp-20130509/ippTools/src/chiptool.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ippTools/src/chiptool.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippTools/src/chiptool.c (revision 35747)
@@ -493,5 +493,5 @@
psFree(where);
- psStringAppend(&query, "\nORDER BY priority DESC, chip_id");
+ psStringAppend(&query, "\nORDER BY priority DESC, chip_id, class_id");
// treat limit == 0 as "no limit"
Index: /branches/eam_branches/ipp-20130509/ippTools/src/pstamptool.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ippTools/src/pstamptool.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippTools/src/pstamptool.c (revision 35747)
@@ -142,7 +142,4 @@
PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval",false, false);
- PXOPT_LOOKUP_BOOL(no_magic, config->args, "-set_no_magic", false);
-
- bool need_magic = ! no_magic;
if (!pstampDataStoreInsert(config->dbh,
@@ -155,5 +152,5 @@
uri,
pollInterval,
- need_magic
+ false
)) {
psError(PS_ERR_UNKNOWN, false, "database error");
Index: /branches/eam_branches/ipp-20130509/ippTools/src/pstamptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ippTools/src/pstamptoolConfig.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ippTools/src/pstamptoolConfig.c (revision 35747)
@@ -51,5 +51,4 @@
psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_label", 0, "define datastore label", NULL);
psMetadataAddS32(adddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 60);
- psMetadataAddBool(adddatastoreArgs,PS_LIST_TAIL, "-set_no_magic", 0, "set magic not required", true);
// -datastore
Index: /branches/eam_branches/ipp-20130509/ppImage/src/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/Makefile.am (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/Makefile.am (revision 35747)
@@ -66,5 +66,6 @@
ppImageAddNoise.c \
ppImageRandomGaussian.c \
- ppImageAuxiliaryMask.c
+ ppImageAuxiliaryMask.c \
+ ppImageBurntoolApply.c
CLEANFILES = *~
Index: /branches/eam_branches/ipp-20130509/ppImage/src/burnparams.h
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/burnparams.h (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/burnparams.h (revision 35747)
@@ -0,0 +1,59 @@
+/* A global buffer into which routines can accumulate detections */
+#ifndef _INCLUDED_burnparams_
+#define _INCLUDED_burnparams_
+
+/* '#define EXTERN' in just one file, burntool.c, to declare variables */
+#ifndef EXTERN
+#define EXTERN extern
+#endif
+
+#define MAXBURN 10000
+EXTERN OBJBOX boxbuf[MAXBURN];
+
+EXTERN int *median_buf; /* Generic buffer for integer medians */
+EXTERN int nmedian_buf;
+
+EXTERN MTYPE *mbuf; /* Buffer for object mask */
+EXTERN int nmbuf;
+
+EXTERN MTYPE *msbuf; /* Buffer for vetoed star mask */
+EXTERN int nmsbuf;
+
+EXTERN DTYPE *imbuf; /* Copy of cell data w/o BZERO */
+EXTERN int nimbuf;
+
+EXTERN int VERBOSE; /* Verbosity level */
+
+EXTERN int BZERO;
+EXTERN int USHORT_BIAS; /* Bias level restored to ushort stamps */
+
+EXTERN int MAX_READ_NOISE; /* Maximum believable read noise (ADU) */
+EXTERN double MIN_EADU; /* Minimum believable e/ADU */
+EXTERN int SAT4SURE; /* Ignore pixels above for noise estimate */
+EXTERN double MIN_BLAST_PASS; /* Allow blasted cells if they have BIG satfrac */
+EXTERN double MAX_BLAST_PASS; /* But not if it's all wiped out! */
+
+EXTERN int BURN_THRESH ; /* Threshold for onset of burning */
+EXTERN int TRAIL_THRESH ; /* Trailing might go this low */
+EXTERN int MAX_THRESH ; /* Possibly trailing stars? */
+EXTERN int STAR_THRESH ; /* Threshold for star above sky */
+EXTERN double STAR_FRAC ; /* Fraction to follow star profile */
+EXTERN int PSF_THRESH ; /* Threshold for a star to be a PSF */
+
+//EXTERN double XMASK_GROW ; /* Growth of burned boxes in x dir */
+//EXTERN double YMASK_GROW ; /* Growth of burn/star in y dir */
+
+EXTERN double BMASK_GROW ; /* Growth of burned boxes in size */
+EXTERN double RMASK_GROW ; /* Growth of burn/star in diameter */
+
+EXTERN int MIN_PSF_SIZE ; /* Min box size for stamp selection */
+EXTERN int PSF_CTR_TOL ; /* Choose max or box ctr for stamp */
+EXTERN int CONCAT_FITS; /* Write concat FITS for PSF? (else 3D) */
+EXTERN int MAX_PSF_PER_CELL ; /* Max number of PSF stars accepted per cell */
+
+EXTERN double NEGLIGIBLE_TRAIL; /* Don't sweat less than this * sigma */
+EXTERN int EXPIRE_TRAIL_TIME; /* Expire trails after this interval */
+
+EXTERN int PERSIST_RETAIN; /* Retain bad-slope persistence fits */
+
+#endif /* _INCLUDED_burnparams_ */
Index: /branches/eam_branches/ipp-20130509/ppImage/src/burntool.h
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/burntool.h (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/burntool.h (revision 35747)
@@ -0,0 +1,229 @@
+#ifndef _INCLUDED_burntool_
+#define _INCLUDED_burntool_
+
+/* Burn correction routines:
+ -------------------------
+ * 1. Identify burned spots from saturation threshold
+ ? 2. Identify burns and trails morphologically?
+ ? 2.1 Look for flat-topped stars
+ ? 2.2 Look for asymmetric, smooth, declining trails?
+ ? 2.3 Use y overscan?
+ ? 2.4 Bin in x?
+ * 3. Mask stars and other cruft?
+ * 4. Fit up and down row functions
+ * 5. Read burn time list
+ * 6. Write updated burn time list
+ * 8. Undo subtraction
+ 7. Create and save burn correction FITS table
+*/
+/* Algorithm:
+ ----------
+ 1. Find new burned spots
+ 2. Create new burn entries: x1,x2,y,time=now
+ 3. For each burn:
+ if t = now detrail up
+ else deburn down
+ if fit negligible delete burn
+ else save burn fits
+ */
+/* ToDo:
+ -----
+ * clean up fit start points, esp for persistence
+ * change burn= and persist= to trailin= and trailout=
+ * trim edges of fits
+ * implement MASK_SAT
+ > test for garbage fits (positive slope, etc)
+ * test nearly saturated stars for burn.
+ * undo subtraction (almost perfect -- occasional pixels and baddies)
+ * star gather into 3D FITS
+ * test on donuts
+ * stamp extraction center on box, not max
+ * Add back nominal bias; output as ushort
+ * man page... ongoing...
+ * Scale uses peak/sum?
+ * Change psf coords to center; save in header not pixels
+ x better trail function? A tuned ln(x+A) is better, but too hard.
+ * why various restorations imperfect? edge of cell?
+ * cell mask argument
+... why the honker star not ID'ed as a burn? various other gotchas...
+ * age away old persistence streaks; merge persistence streaks
+ * PSF info on median psf
+ * clean up sky/rms calculation with 5*quartile clipping
+ * change VERBOSE to a bitmask for access to various portions of the code
+ * add access to a lot of the tuning variables...
+ * update man page
+ tuning, tuning, tuning...
+ add correction FITS table to MEF
+*/
+
+#define MAX(a,b) (((a) > (b)) ? (a) : (b))
+#define MIN(a,b) (((a) < (b)) ? (a) : (b))
+#define NINT(x) (x<0?(int)((x)-0.5):(int)((x)+0.5))
+#define ABS(a) (((a) > 0) ? (a) : -(a))
+
+// #define STATIC static /* To make function declarations static */
+#define STATIC /* To make function declarations global */
+
+#define MAXCELL 64 /* Max cells in an OTA */
+#define MAXSIZE 2048 /* Maximum vertical cell size */
+
+#define STAR_RADIUS 4 /* Radius over which a star ctr must be max */
+#define SKY_MARG 3 /* Horiz offset from burn area for sky */
+#define FIT_EDGE 10 /* How far beyond saturation to start fit? */
+#define Y_SCALE 0.01 /* Scale factor for y in fits */
+
+typedef signed short int IMTYPE;/* Data type of image */
+
+typedef int DTYPE; /* Data type of data copy */
+#define NODATA 0 /* Marker for *No Data */
+
+/* Mask codes: keep in order of severity for psf_select()! */
+typedef int MTYPE; /* Data type of mask */
+#define MASK_NONE 0 /* Unmasked pixel (MUST be 0) */
+#define MASK_STAR_HALO 1 /* Extended halo box from star */
+#define MASK_SAT_HALO 2 /* Extended halo box from grow_mask() */
+#define MASK_CTR 3 /* Center mask from star_detect() */
+#define MASK_SAT 4 /* Center mask of saturated pixels */
+
+/* Enumeration of function choices */
+#define FUNC_NONE 0 /* No associated function */
+#define BURN_PWR 1 /* Power law */
+#define BURN_EXP 2 /* Exponential */
+#define BURN_BLASTED 3 /* Blasted top to bottom: flag only for IPP */
+#define BURN_POSSLOPE 4 /* Positive slope fit (bad) but significant */
+#define PSF_STAR 9 /* Unfitted: good psf star */
+
+/* Fit error codes */
+#define FIT_ERROR 1 /* linearfit failed */
+#define FIT_TOP_ERROR 2 /* Saturation extends to top: no points */
+#define FIT_SLOPE_ERROR 3 /* Unreasonable fit */
+#define FIT_ALL_GONE 4 /* No column survived as significant */
+#define FIT_EXPIRED 9 /* Don't carry any more as persistent */
+
+/* Fit parameters */
+#define FIT_MIN_SLOPE -5.0 /* minimum slope which is a credible fit */
+#define FIT_MAX_SLOPE 0.0 /* maximum slope which is a credible fit */
+
+/* Verbosity bits */
+#define VISTAMARKER "/tmp/markem.pro" /* Write Vista pro to mark stars?*/
+#define VERB_NORM 0x0001 /* Normal, verbose output */
+#define VERB_DETECT 0x0002 /* Dump detection process */
+#define VERB_PSFSEL 0x0004 /* Dump out PSF selection process */
+#define VERB_FIT 0x0008 /* Dump fit progress */
+#define VERB_FITPROF 0x0010 /* Dump fit profiles */
+#define VERB_VISTA 0x0020 /* Write vista markers as /tmp/markem.pro */
+#define VERB_BOXGROW 0x0040 /* Dump box growth diagnostics */
+#define VERB_MASK 0x0080 /* Write mask in place of corrected image */
+
+/* Description of a potentially burned trail */
+typedef struct obj_box {
+ int cell; /* what cell is this one in? */
+ int time; /* PON time when it was created */
+ int sx; /* left corner */
+ int sy; /* bottom corner */
+ int ex; /* right corner */
+ int ey; /* top corner */
+ int cx; /* center x (position of max) */
+ int cy; /* center y (position of max) */
+ int max; /* max data value (above sky) */
+ int y0m; /* min y value at sx */
+ int y0p; /* max y value at sx */
+ int y1m; /* min y value at ex */
+ int y1p; /* max y value at ex */
+ int x0m; /* min x value at sy */
+ int x0p; /* max x value at sy */
+ int x1m; /* min x value at ey */
+ int x1p; /* max x value at ey */
+ int sat; /* saturated (i.e. bigger than BURNTHRESH)? */
+ int func; /* what are we going to do about it? */
+ int diff; /* median diff of up minus down (sum for stars) */
+ int up; /* does it trail up or down? (stamp sx for stars) */
+ int y0; /* y origin for the fit (stamp sy for stars) */
+ int burned; /* do we think it's left a burn trail? */
+ int fiterr; /* error fitting the trail? */
+ int sxfit; /* starting column for fits */
+ int exfit; /* ending column for fits */
+ int nfit; /* how many columns were corrected? */
+ IMTYPE *stamp; /* postage stamp of this object */
+ int *xfit; /* x of each value of the start of correction */
+ int *yfit; /* y value of the start of correction */
+ double slope; /* slope of fit (stamp sum/max) */
+ double *zero; /* zero of fit for each of the columns */
+} OBJBOX;
+
+/* Info for an entire cell */
+typedef struct cell_info {
+ int cell; /* Cell number */
+ int bias; /* Bias level */
+ int sky; /* Sky level */
+ int rms; /* RMS in the sky */
+ int time; /* PON time of this cell */
+ double satfrac; /* Fraction of SAT4SURE saturated pixels */
+ int nburn; /* Number of trails left by sat stars */
+ OBJBOX *burn; /* Stars which we think left a trail */
+ int npersist; /* Number of old persistence streaks */
+ OBJBOX *persist; /* Persistent streaks */
+ int nstar; /* Number of stars */
+ OBJBOX *star; /* Stars, harmless we think */
+} CELL;
+
+/* Prototypes */
+STATIC int mem_init(int nx, int ny, int NX, int NY);
+STATIC int burn_fix(int nx, int ny, int stride, int NY, IMTYPE *buf,
+ CELL *cell, int cellnum);
+STATIC int burn_test(int nx, int ny, int NX, DTYPE *data, int rms,
+ MTYPE *mask, OBJBOX *box);
+STATIC int burn_restore(int nx, int ny, int NX, IMTYPE *buf, CELL *cell);
+STATIC int burn_apply(int nx, int ny, int NX, IMTYPE *buf, CELL *cell);
+STATIC int persist_read(CELL *cell, const char *infile, int apply);
+STATIC int persist_write(CELL *cell, const char *outfile);
+STATIC int persist_fix(int nx, int ny, int stride, IMTYPE *buf, CELL *cell);
+STATIC int persist_merge(CELL *cell);
+
+//fh_result persist_fits_read(CELL *cell, const char *filename, int apply);
+//fh_result persist_fits_write(CELL *cell, HeaderUnit phu);
+//fh_result persist_fits_remove_tables(HeaderUnit phu_in, const char *fileout);
+
+STATIC int star_detect(int nx, int ny, int NX, int NY, DTYPE *data,
+ MTYPE *mask, MTYPE *veto, CELL *cell, int cellnum);
+STATIC int burn_check(int nx, int ny, int stride, int NY, DTYPE *buf,
+ MTYPE *mask, CELL *cell);
+
+STATIC int cell_stats(int nx, int ny, int NX, int NY, DTYPE *data, CELL *cell);
+STATIC int burn_blab(CELL *cell);
+STATIC int persist_blab(CELL *cell);
+STATIC int vista_marker(CELL *cell, char *fname);
+
+STATIC int fit_trail(int nx, int ny, int NX, DTYPE *data, MTYPE *mask,
+ OBJBOX *box, int up, int sky, int rms, int fitfunc);
+STATIC int sub_fit(int nx, int ny, int NX, IMTYPE *buf, OBJBOX *box, int sign);
+
+STATIC int psf_select(int nx, int ny, int NX, MTYPE *mask, DTYPE *buf,
+ int nbox, OBJBOX *box, int size, int sky);
+STATIC int psf_write(int nx, int ny, CELL *OTA, int otanum, const char *psffile);
+STATIC int psf_write_stats(int nx, int ny, CELL *OTA, int otanum, const char *statfile, int psfavg);
+STATIC int psf_stats(int nx, int ny, IMTYPE *data, int bias,
+ double *fwhm, double *q);
+
+STATIC int grow_box(int nx, int ny, int NX, DTYPE *data, int thr, OBJBOX *box);
+STATIC int grow_mask(int nx, int ny, int NX, DTYPE *mask, double xfac,
+ double yfac, int maskval, int nbox, OBJBOX *box);
+STATIC int local_max(int i, int j, int r, int nx, int ny, int NX, DTYPE *data);
+
+STATIC int wlinearfit(int npt, double *x, double *y,
+ double *w, double *a, double *b);
+STATIC int linearrms(int npt, double *x, double *y, double a, double b, double *rms);
+STATIC int int_median(int n, int *key);
+STATIC double double_median(int n, double *key);
+
+STATIC int qsort_int(int n, int *key);
+STATIC double qsort_dbl(int n, double *key);
+
+STATIC void syntax(const char *prog);
+
+int write_2dfits(int nx, int ny, int sx, int sy, IMTYPE *data, int fd);
+int write_3dhdr(int nx, int ny, int nz, int ncmt, char *cmt[], int fd);
+int write_2ddata(int nx, int ny, int *ntot, IMTYPE *data, int fd);
+int write_3dend(int *ntot, int fd);
+
+#endif /* _INCLUDED_burntool_ */
Index: /branches/eam_branches/ipp-20130509/ppImage/src/ppImage.h
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/ppImage.h (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/ppImage.h (revision 35747)
@@ -26,4 +26,5 @@
bool doMaskBuild; // Build internal mask
bool doVarianceBuild; // Build internal variance map
+ bool doApplyBurntool; // apply burntool correction
bool doMaskBurntool; // mask potential burntool trails
bool doMaskSat; // mask saturated pixels
@@ -182,5 +183,8 @@
bool ppImageCheckNoise(pmConfig *config, ppImageOptions *options, pmFPAview *view);
-bool ppImageBurntoolMask(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *mask);
+bool ppImageBurntoolMask(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout);
+bool ppImageBurntoolMaskFromTable(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout);
+
+bool ppImageBurntoolApply(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout);
bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, const ppImageOptions *options);
Index: /branches/eam_branches/ipp-20130509/ppImage/src/ppImageArguments.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/ppImageArguments.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/ppImageArguments.c (revision 35747)
@@ -117,4 +117,10 @@
pmConfigFileSetsMD (config->arguments, &argc, argv, "LINEARITY", "-linearity", "-linearlist");
+ if ((argnum = psArgumentGet(argc, argv, "-burntool"))) {
+ psArgumentRemove(argnum, &argc, argv);
+ psMetadataAddStr(config->arguments, PS_LIST_TAIL, "BURNTOOL.TABLE", PS_META_REPLACE, "", argv[argnum]);
+ psArgumentRemove(argnum, &argc, argv);
+ }
+
// chip selection is used to limit chips to be processed
if ((argnum = psArgumentGet (argc, argv, "-chip"))) {
Index: /branches/eam_branches/ipp-20130509/ppImage/src/ppImageAuxiliaryMask.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/ppImageAuxiliaryMask.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/ppImageAuxiliaryMask.c (revision 35747)
@@ -60,5 +60,5 @@
if (!options->doAuxMask) {
- psLogMsg ("ppImage", PS_LOG_INFO, "Auxiliary mask not enabled.");
+ psLogMsg ("ppImage", PS_LOG_DETAIL, "Auxiliary mask not enabled.");
return true;
}
Index: /branches/eam_branches/ipp-20130509/ppImage/src/ppImageBurntoolApply.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/ppImageBurntoolApply.c (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/ppImageBurntoolApply.c (revision 35747)
@@ -0,0 +1,557 @@
+/*
+ * ppImageBurntoolApply.c
+ * Includes code ported from gpcsw/gpcsrc to read burntool table files and apply the fits
+ */
+
+#include "ppImage.h"
+
+// Now include the files copied from gpcsw/gpcsrc
+
+// these get redefined
+#undef MAX
+#undef MIN
+
+#include "burntool.h"
+#define EXTERN /* define EXTERN to declare variables in params.h */
+#include "burnparams.h"
+
+
+static void * getBurntoolDataForCell(pmConfig *config, ppImageOptions *options, int burntool_cell);
+
+bool ppImageBurntoolApply(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout)
+{
+ // convert from pmFPA's cell number to burntool's notion
+ int burntool_cell = (view->cell % 8) * 8 + (view->cell - (view->cell % 8)) / 8;
+
+ psString cellName = psMetadataLookupStr(NULL, readout->parent->concepts, "CELL.NAME");
+ (void) cellName;
+
+ CELL *burntoolData = getBurntoolDataForCell(config, options, burntool_cell);
+ if (!burntoolData) {
+ psError(PS_ERR_UNKNOWN, false, "Unable to find burntool data for cell");
+ return false;
+ }
+
+ psImage *image = readout->image;
+
+#ifdef DUMP_CELL_IMAGES
+ psString dumpPath = NULL;
+ psStringAppend(&dumpPath, "cellimages/%s.before.fits", cellName);
+ psphotSaveImage(NULL, image, dumpPath);
+ psFree(dumpPath);
+#endif
+
+ // The following code is adapted from burntool's functions burn_apply() which is in burn_fix.c and
+ // sub_fit() which was located in trailfit.c
+
+ int ny = image->numRows;
+
+ psF32 satVal = psMetadataLookupF32(NULL, readout->parent->concepts, "CELL.SATURATION");
+
+ for (int p = 0; p < burntoolData->npersist; p++) {
+ OBJBOX *box = &(burntoolData->persist[p]);
+
+ if(box->fiterr) continue;
+ if(box->func != BURN_PWR && box->func != BURN_EXP) {
+ continue;
+ }
+
+ int i, j, k;
+ int y0, y1, ys, dy;
+
+ y0 = box->y0;
+ y1 = (box->up) ? ny-1 : 0;
+ dy = (y0up) {
+ ys = box->sy;
+ } else {
+ ys = box->ey;
+ }
+
+ /* Calculate all needed y values */
+ double ybuf[MAXSIZE];
+ for(j=ys; dy*j<=dy*y1; j+=dy) {
+ if(box->func == BURN_PWR) {
+ if(dy*(j-y0) <= 0) {
+ ybuf[j] = exp(box->slope*log(Y_SCALE*1.0));
+ } else {
+ ybuf[j] = exp(box->slope*log(Y_SCALE*dy*(j-y0)));
+ }
+ } else if(box->func == BURN_EXP) {
+ // ybuf[j] = exp(box->slope*Y_SCALE*MAX(0,dy*(j-y0)));
+ ybuf[j] = exp(box->slope*Y_SCALE*dy*(j-y0));
+ }
+ }
+
+
+ /* Subtract the fits */
+ // printf("%3d %3d %3d %1d %1d %8.5f\n",
+ // box->cx, box->cy, box->y0, box->func, box->up, box->slope);
+
+ for(k=0; knfit; k++) {
+ i = box->xfit[k];
+ for(j=box->yfit[k]; dy*j<=dy*y1; j+=dy) {
+ psF32 inputPixel = image->data.F32[j][i];
+ if (false && inputPixel >= satVal) {
+ // don't modify saturated pixels
+ continue;
+ }
+
+#ifndef MATCH_BURNTOOL
+ psF32 delta = box->zero[k] * ybuf[j];
+ image->data.F32[j][i] -= delta;
+#else
+ // do math in 16 bit signed int space in order to match how the burntool program
+ // behaves with regard to underflows. burntool allows underflows so that the burntool
+ // application is reversible.
+ // Verified that the image matches the burntool implementation with this code
+ // which shows that we are reading and processing the table correctly.
+ int delta = box->zero[k] * ybuf[j] + 0.5;
+ psS16 pixel;
+ // adjust for BZERO
+ if (inputPixel >= 65535) {
+ pixel = 32767;
+ } else if (inputPixel <= 0) {
+ pixel = -32768;
+ } else {
+ pixel = inputPixel - 32768;
+ }
+ pixel -= delta;
+ // replace the pixel with correction applied
+ image->data.F32[j][i] = (psF32) pixel + 32768;
+#endif
+ }
+ }
+ }
+#ifdef DUMP_CELL_IMAGES
+ dumpPath = NULL;
+ psStringAppend(&dumpPath, "cellimages/%s.after.fits", cellName);
+ psphotSaveImage(NULL, image, dumpPath);
+ psFree(dumpPath);
+#endif
+
+ return true;
+}
+
+
+bool burntoolFileRead = false;
+static CELL OTA[MAXCELL]; // cell structure for entire OTA
+static void * getBurntoolDataForCell(pmConfig *config, ppImageOptions *options, int burntool_cell)
+{
+ if (!burntoolFileRead) {
+ psString burntoolTablePath = psMetadataLookupStr(NULL, config->arguments, "BURNTOOL.TABLE");
+ if (!burntoolTablePath) {
+ psError(PS_ERR_UNKNOWN, false, "failed to lookup BURNTOOL.TABLE in arguments");
+ return NULL;
+ }
+ psString burntoolFileName = pmConfigConvertFilename(burntoolTablePath, config, false, false);
+ if (!burntoolFileName) {
+ psError(PS_ERR_UNKNOWN, false, "failed to convert filename for %s", burntoolTablePath);
+ return NULL;
+ }
+ if (persist_read(OTA, burntoolFileName, 1)) {
+ psError(PS_ERR_UNKNOWN, "true", "failed to read burntool file");
+ return NULL;
+ }
+ psFree(burntoolFileName);
+ burntoolFileRead = true;
+ }
+
+ return (void *) &OTA[burntool_cell];
+}
+#define PPIMAGE_BURNTOOL_DEBUG 0
+
+// adapted from ppImageBurntoolMask
+bool ppImageBurntoolMaskFromTable(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *mask)
+{
+ bool status = true;
+ int burntool_cell;
+
+ /* Redirects and Memory juggling. */
+ view->readout = 0;
+ psImage *image = mask->mask;
+
+ /* Set the maskValue from the recipes. */
+ psImageMaskType maskValue = options->burntoolMask;
+
+ burntool_cell = (view->cell % 8) * 8 + (view->cell - (view->cell % 8)) / 8;
+ CELL *burntoolData = getBurntoolDataForCell(config, options, burntool_cell);
+ if (!burntoolData) {
+ psError(PS_ERR_UNKNOWN, false, "Unable to find burntool data for cell");
+ return false;
+ }
+
+#if PPIMAGE_BURNTOOL_DEBUG
+ psLogMsg("ppImageBurntoolMask", 4, "Status: %d %d\n",burntoolData->npersist,maskValue);
+#endif
+
+ psLogMsg("ppImageBurntoolMask", 4, "Cell mapping: %d %d %d\n",view->cell,burntool_cell,-1);
+ for (int row = 0; row < burntoolData->npersist; row++) {
+ OBJBOX *box = &(burntoolData->persist[row]);
+ if (((options->burntoolTrails & 0x01)&& (box->func == 4))||
+ (((options->burntoolTrails & 0x02)&& (box->up == 1))||
+ ((options->burntoolTrails & 0x04)&& (box->up == 0)))) {
+ /* If the fit fails, burntool reports zero here. This
+ signifies that it expected to see a trail (else why
+ fit) but did not find it when it attempted to
+ correct. */
+#if PPIMAGE_BURNTOOL_DEBUG
+ psLogMsg ("ppImageBurntoolMask", 4, "Masking! %d (%d %d %d) %d %d",
+ burntool_cell,
+ ((options->burntoolTrails & 0x0001)&& (box->nfit == 0)),
+ ((options->burntoolTrails & 0x02)&& (box->up == 1)),
+ ((options->burntoolTrails & 0x04)&& (box->up == 0)),
+ options->burntoolTrails,
+ maskValue
+ );
+#endif
+ for (int i = box->sxfit; i<= box->exfit; i++) {
+
+ if (box->up == 0) {
+ for (int j = 0; j <= box->y1p; j++) {
+#if PPIMAGE_BURNTOOL_DEBUG
+ psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n",
+ i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue);
+#endif
+ image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue;
+ }
+ } else {
+ for (int j = box->y1m; j < image->numRows ; j++) {
+#if PPIMAGE_BURNTOOL_DEBUG
+ psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n",
+ i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue);
+#endif
+ image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue;
+ }
+ }
+ }
+ }
+ }
+
+ return(status);
+}
+
+/* From persistio.c - read and write persistence info */
+/****************************************************************/
+/* persist_read(): Read all the persistence trails from a file */
+STATIC int persist_read(CELL *cell, const char *infile, int apply)
+{
+ int i, k, nbox=0;
+ char line[1024];
+ FILE *fp;
+
+/* Initialize the counts */
+ for(k=0; k 0) {
+ boxbuf[nbox].zero = (double *)calloc(boxbuf[nbox].nfit, sizeof(double));
+ boxbuf[nbox].xfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int));
+ boxbuf[nbox].yfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int));
+ if(boxbuf[nbox].zero == NULL ||
+ boxbuf[nbox].xfit == NULL ||
+ boxbuf[nbox].yfit == NULL) {
+ fprintf(stderr, "\rerror: failed to alloc boxbuf\n");
+ exit(-673);
+ }
+
+ for(i=0; i= MAXCELL) {
+ fprintf(stderr, "\rerror: illegal cell %d at %d from '%s'\n",
+ k, nbox, line);
+ fclose(fp);
+ return(-1);
+ }
+ cell[k].npersist += 1;
+
+ if(++nbox >= MAXBURN) {
+ fprintf(stderr, "\rerror: persist_read overflowed MAXBURN\n");
+ fclose(fp);
+ return(-1);
+ }
+ }
+
+/* Allocate some space for them */
+ for(k=0; k 0) {
+ if( (cell[k].persist = (OBJBOX *)calloc(i, sizeof(OBJBOX))) == NULL) {
+ fprintf(stderr, "\rerror: failed to alloc cell persist buffer\n");
+ exit(-674);
+ }
+ cell[k].npersist = 0;
+ }
+ }
+/* Copy the results to the cells */
+ for(i=0; inpersist;
+ box = cell->persist;
+
+/* Assess the clustering of all the streaks */
+ for(i=0; i= MAXSIZE-1) continue;
+ for(i=box[k].sxfit; i<=box[k].exfit; i++) xusage[i] += 1;
+ }
+
+/* Identify clusters */
+ for(xs=0; xs xe || box[k].exfit < xs) continue;
+ boxid[n] = k;
+ yctr[n] = (box[k].y0m+box[k].y1m+box[k].y0p+box[k].y0p+2) / 4;
+ n++;
+ }
+
+/* Case 1: different y start => sever */
+ for(kp=0; kp 0) zk = box[k].zero[box[k].nfit/2];
+ for(jp=kp+1; jp 0) zj = box[j].zero[box[j].nfit/2];
+ if(ABS(yctr[jp]-yctr[kp]) > DIFFERENT_STREAK) {
+/* Trim back the feebler streak */
+ if(zk > zj) {
+ if(box[j].sxfit >= box[k].sxfit)
+ box[j].sxfit = MAX(box[j].sxfit, box[k].exfit+1);
+ if(box[j].exfit <= box[k].exfit)
+ box[j].exfit = MIN(box[j].exfit, box[k].sxfit-1);
+ } else {
+ if(box[k].sxfit >= box[j].sxfit)
+ box[k].sxfit = MAX(box[k].sxfit, box[j].exfit+1);
+ if(box[k].exfit <= box[j].exfit)
+ box[k].exfit = MIN(box[k].exfit, box[j].sxfit-1);
+ }
+ }
+ }
+ }
+
+/* Case 2: pretty much the same y start => union */
+ for(kp=0; kp exfit) */
+ for(k=0; k box[k].exfit) {
+ for(j=k; jnpersist = nbox;
+
+
+ return(0);
+}
+
+
+/****************************************************************/
+/* persist_write(): Write all the persistence data for the next image */
+STATIC int persist_write(CELL *cell, const char *outfile)
+{
+ int i, k, j, err=0;
+ FILE *fp;
+
+ if(outfile == NULL) return(0);
+ if( (fp=fopen(outfile, "w")) == NULL) {
+ fprintf(stderr, "\rerror: cannot open '%s' for writing\n", outfile);
+ return(-1);
+ }
+
+/* Dump out all the burns for the next image... */
+/* FIXME: what to do about generic cell info? */
+ fprintf(fp, "# Cell: %d sky= %d rms= %d bias= %d\n",
+ cell[0].cell, cell[0].sky, cell[0].rms, cell[0].bias);
+ fprintf(fp, "#Cell time cx cy max y0 sx sy ex ey y0m y0p y1m y1p x0m x0p x1m x1p F up slope nfit sxf exf\n");
+
+ for(j=0; j EXPIRE_TRAIL_TIME)
+ continue;
+
+ if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+ if(cell[j].persist[k].fiterr != FIT_SLOPE_ERROR) {
+ if(cell[j].persist[k].fiterr) continue;
+ if(cell[j].persist[k].nfit <= 0) continue;
+ }
+ } else {
+ if(cell[j].persist[k].fiterr) continue;
+ if(cell[j].persist[k].nfit <= 0) continue;
+ }
+ fprintf(fp, "%3d %7d %3d %3d %5d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %1d %1d %9.6f %3d %3d %3d %d\n",
+ j, cell[j].persist[k].time,
+ cell[j].persist[k].cx, cell[j].persist[k].cy,
+ cell[j].persist[k].max, cell[j].persist[k].y0,
+ cell[j].persist[k].sx, cell[j].persist[k].sy,
+ cell[j].persist[k].ex, cell[j].persist[k].ey,
+ cell[j].persist[k].y0m, cell[j].persist[k].y0p,
+ cell[j].persist[k].y1m, cell[j].persist[k].y1p,
+ cell[j].persist[k].x0m, cell[j].persist[k].x0p,
+ cell[j].persist[k].x1m, cell[j].persist[k].x1p,
+ cell[j].persist[k].func, cell[j].persist[k].up,
+ cell[j].persist[k].slope, cell[j].persist[k].nfit,
+ cell[j].persist[k].sxfit, cell[j].persist[k].exfit,
+ cell[j].persist[k].fiterr);
+ for(i=0; idoMaskBuild) {
@@ -74,6 +74,19 @@
pmMaskBadPixels(input, mask, options->maskValue);
}
+
+ if (options->doApplyBurntool)
+ {
+ // extern bool ppImageBurntoolApply(pmConfig *, ppImageOptions *, pmFPAview *, pmReadout *);
+ ppImageBurntoolApply(config, options, view, input);
+ }
+
if (options->doMaskBurntool) {
- ppImageBurntoolMask(config,options,view,input);
+ if (options->doApplyBurntool) {
+ // build burntool mask from data input burntool table
+ ppImageBurntoolMaskFromTable(config,options,view,input);
+ } else {
+ // build burntool mask from data in the input image's fits extension
+ ppImageBurntoolMask(config,options,view,input);
+ }
}
Index: /branches/eam_branches/ipp-20130509/ppImage/src/ppImageDetrendRecord.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/ppImageDetrendRecord.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/ppImageDetrendRecord.c (revision 35747)
@@ -76,4 +76,7 @@
detrendRecord(options->doNonLin, detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN", "Non-linearity table filename");
+
+ detrendRecord(options->doDark & options->useVideoDark, detrend, config, view, "PPIMAGE.VIDEODARK", "DETREND.VIDEODARK", "VideoDark filename");
+ detrendRecord(options->doMask & options->useVideoMask, detrend, config, view, "PPIMAGE.VIDEOMASK", "DETREND.VIDEOMASK", "VideoMASK filename");
psFree (detrend);
return true;
Index: /branches/eam_branches/ipp-20130509/ppImage/src/ppImageOptions.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/ppImageOptions.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/ppImageOptions.c (revision 35747)
@@ -23,4 +23,5 @@
options->doMaskLow = false; // mask low pixels
options->doMaskBurntool = false; // mask potential burntool trails
+ options->doApplyBurntool = false; // apply burntool correction
options->doVarianceBuild = false; // Build internal variance
options->doMask = false; // Mask bad pixels
@@ -231,4 +232,5 @@
options->doMaskLow = psMetadataLookupBool(NULL, recipe, "MASK.LOW");
options->doMaskBurntool = psMetadataLookupBool(NULL, recipe, "MASK.BURNTOOL");
+ options->doApplyBurntool = psMetadataLookupBool(NULL, recipe, "APPLY.BURNTOOL");
options->doVarianceBuild = psMetadataLookupBool(NULL, recipe, "VARIANCE.BUILD");
options->doAuxMask = psMetadataLookupBool(NULL, recipe, "MASK.AUXMASK");
Index: /branches/eam_branches/ipp-20130509/ppImage/src/ppImageParseCamera.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppImage/src/ppImageParseCamera.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/ppImage/src/ppImageParseCamera.c (revision 35747)
@@ -184,4 +184,12 @@
}
outCTE->save = true;
+ }
+ if (options->doApplyBurntool) {
+ // If no burntool file was supplied do not fail. (camera_exp.pl does not supply it
+ // for example.
+ if (!psMetadataLookupStr(NULL, config->arguments, "BURNTOOL.TABLE")) {
+ psWarning("BURNTOOL.TABLE not supplied setting doApplyBurntool to false");
+ options->doApplyBurntool = false;
+ }
}
Index: /branches/eam_branches/ipp-20130509/psastro/src/psastro.h
===================================================================
--- /branches/eam_branches/ipp-20130509/psastro/src/psastro.h (revision 35746)
+++ /branches/eam_branches/ipp-20130509/psastro/src/psastro.h (revision 35747)
@@ -83,5 +83,5 @@
bool psastroOneChipGrid (pmFPA *fpa, pmChip *chip, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
bool psastroOneChipFit (pmFPA *fpa, pmChip *chip, pmReadout *readout, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
-bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source);
+bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source, bool saveExistingMatchedRefs);
bool psastroRefstarSubset (pmReadout *readout);
pmLumFunc *psastroLuminosityFunction (psArray *stars, pmLumFunc *rawFunc);
Index: /branches/eam_branches/ipp-20130509/psastro/src/psastroAnalysis.c
===================================================================
--- /branches/eam_branches/ipp-20130509/psastro/src/psastroAnalysis.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/psastro/src/psastroAnalysis.c (revision 35747)
@@ -86,5 +86,7 @@
}
- if (!psastroChooseRefstars (config, refs, "PSASTRO.INPUT")) {
+ bool skipastro = psMetadataLookupBool (&status, config->arguments, "PSASTRO.SKIP.ASTRO");
+
+ if (!psastroChooseRefstars (config, refs, "PSASTRO.INPUT", skipastro)) {
psError (PSASTRO_ERR_UNKNOWN, false, "failed to select reference data for chips\n");
psFree(refs);
@@ -108,5 +110,10 @@
mosastro = psMetadataLookupBool (&status, recipe, "PSASTRO.MOSAIC.MODE");
}
- if (!chipastro && !mosastro) {
+
+ if (skipastro) {
+ chipastro = false;
+ mosastro = false;
+ psLogMsg ("psastro", 3, "skip astrometry mode, accepting input astrometry\n");
+ } else if (!chipastro && !mosastro) {
psLogMsg ("psastro", 3, "no astrometry mode selected, assuming chip astrometry\n");
chipastro = true;
@@ -144,12 +151,14 @@
}
- if (!psastroZeroPoint (config)) {
- psError(psErrorCodeLast(), false, "Failed to calculate zero point.");
- return false;
- }
+ if (!skipastro) {
+ if (!psastroZeroPoint (config)) {
+ psError(psErrorCodeLast(), false, "Failed to calculate zero point.");
+ return false;
+ }
- if (!psastroAstromGuessCheck (config)) {
- psError(psErrorCodeLast(), false, "Failed to check astrometry guess.");
- return false;
+ if (!psastroAstromGuessCheck (config)) {
+ psError(psErrorCodeLast(), false, "Failed to check astrometry guess.");
+ return false;
+ }
}
Index: /branches/eam_branches/ipp-20130509/psastro/src/psastroArguments.c
===================================================================
--- /branches/eam_branches/ipp-20130509/psastro/src/psastroArguments.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/psastro/src/psastroArguments.c (revision 35747)
@@ -74,4 +74,5 @@
// apply mosastro mode?
+ bool mosastro = false;
if ((N = psArgumentGet (argc, argv, "-mosastro"))) {
psArgumentRemove (N, &argc, argv);
@@ -81,7 +82,9 @@
psArgumentRemove (N, &argc, argv);
psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.MOSAIC.MODE", PS_META_REPLACE, "", true);
+ mosastro = true;
}
// apply chipastro mode?
+ bool chipastro = false;
if ((N = psArgumentGet (argc, argv, "-chipastro"))) {
psArgumentRemove (N, &argc, argv);
@@ -91,4 +94,18 @@
psArgumentRemove (N, &argc, argv);
psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.CHIP.MODE", PS_META_REPLACE, "", true);
+ chipastro = true;
+ }
+ if ((N = psArgumentGet (argc, argv, "-skipastro"))) {
+ psArgumentRemove (N, &argc, argv);
+ if (mosastro) {
+ psError(PSASTRO_ERR_ARGUMENTS, true, "cannot specify +mosastro with -skipastro");
+ psErrorStackPrint(stderr, "exit");
+ return false;
+ }
+ if (chipastro) {
+ psError(PSASTRO_ERR_ARGUMENTS, true, "cannot specify +chipastro with -skipastro");
+ return false;
+ }
+ psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.SKIP.ASTRO", PS_META_REPLACE, "", true);
}
Index: /branches/eam_branches/ipp-20130509/psastro/src/psastroChooseRefstars.c
===================================================================
--- /branches/eam_branches/ipp-20130509/psastro/src/psastroChooseRefstars.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/psastro/src/psastroChooseRefstars.c (revision 35747)
@@ -20,5 +20,5 @@
}
-bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source) {
+bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source, bool saveExistingMatchedRefs) {
bool status;
@@ -55,5 +55,7 @@
// analysis. however, we are re-doing the astrometry here, so remove
// that prior set of matched references
- psMetadataRemoveKey (fpa->analysis, "MATCHED_REFS");
+ if (!saveExistingMatchedRefs) {
+ psMetadataRemoveKey (fpa->analysis, "MATCHED_REFS");
+ }
}
Index: /branches/eam_branches/ipp-20130509/psastro/src/psastroExtractAnalysis.c
===================================================================
--- /branches/eam_branches/ipp-20130509/psastro/src/psastroExtractAnalysis.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/psastro/src/psastroExtractAnalysis.c (revision 35747)
@@ -44,5 +44,5 @@
}
- if (!psastroChooseRefstars (config, refs, "PSASTRO.EXTRACT.ASTROM")) {
+ if (!psastroChooseRefstars (config, refs, "PSASTRO.EXTRACT.ASTROM", false)) {
psError (PSASTRO_ERR_UNKNOWN, false, "failed to select reference data for chips\n");
psFree(refs);
Index: /branches/eam_branches/ipp-20130509/pstamp/scripts/pstamp_job_run.pl
===================================================================
--- /branches/eam_branches/ipp-20130509/pstamp/scripts/pstamp_job_run.pl (revision 35746)
+++ /branches/eam_branches/ipp-20130509/pstamp/scripts/pstamp_job_run.pl (revision 35747)
@@ -68,5 +68,4 @@
or my_die ("unable to redirect output to $logDest", $job_id, $PS_EXIT_UNKNOWN_ERROR);
}
-
if (!$dbserver) {
Index: /branches/eam_branches/ipp-20130509/pstamp/src/ppstampMakeStamp.c
===================================================================
--- /branches/eam_branches/ipp-20130509/pstamp/src/ppstampMakeStamp.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/pstamp/src/ppstampMakeStamp.c (revision 35747)
@@ -814,4 +814,9 @@
psArray *inSources = inDetections->allSources;
+ // copy analysis metadata in order to pick up any data stored there from the input file.
+ // This includes detection efficiency analysis and extended source parameters.
+ psMetadataCopy(outReadout->analysis, inReadout->analysis);
+
+ // now replace the sources with a new container
pmDetections *outDetections = pmDetectionsAlloc();
psMetadataAddPtr (outReadout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN,
@@ -820,5 +825,6 @@
psArray *outSources = outDetections->allSources = psArrayAllocEmpty( 100 );
- // include sources that are off the stamp by just a little bit
+ // include sources that are off the stamp by just a little bit
+ // XXX: should I be doing this?
#define EXPAND_SIZE 16
psRegion sourceRegion = extractRegion;
@@ -828,4 +834,6 @@
sourceRegion.y1 += EXPAND_SIZE;
+ bool adjustCoords = extractRegion.x0 != 0 || extractRegion.y0 != 0;
+
for (int i = 0; i < inSources->n; i++ ) {
pmSource *src = inSources->data[i];
@@ -837,4 +845,30 @@
continue;
}
+ if (adjustCoords) {
+ // adjust the source position for the stamp coordinate system
+ src->peak->x -= extractRegion.x0;
+ src->peak->xf -= extractRegion.x0;
+
+ src->peak->y -= extractRegion.y0;
+ src->peak->yf -= extractRegion.y0;
+ if (src->moments) {
+ src->moments->Mx -= extractRegion.x0;
+ src->moments->My -= extractRegion.y0;
+ }
+ if (src->modelPSF) {
+ src->modelPSF->params->data.F32[PM_PAR_XPOS] -= extractRegion.x0;
+ src->modelPSF->params->data.F32[PM_PAR_YPOS] -= extractRegion.y0;
+ }
+
+ if (src->modelFits) {
+ for (int j = 0; j < src->modelFits->n; j++) {
+ pmModel *model = src->modelFits->data[j];
+ assert (model != NULL);
+ model->params->data.F32[PM_PAR_XPOS] -= extractRegion.x0;
+ model->params->data.F32[PM_PAR_YPOS] -= extractRegion.y0;
+ }
+ }
+ }
+
psArrayAdd(outSources, 100, src);
}
Index: /branches/eam_branches/ipp-20130509/pswarp/src/pswarpDefineLayout.c
===================================================================
--- /branches/eam_branches/ipp-20130509/pswarp/src/pswarpDefineLayout.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/pswarp/src/pswarpDefineLayout.c (revision 35747)
@@ -40,4 +40,7 @@
// chips are not processed unless we have determined they overlap the inputs
pmFPAExcludeChips (output->fpa);
+ if (bkgModel) {
+ pmFPAExcludeChips (bkgModel->fpa);
+ }
int nInputs = psMetadataLookupS32(&status, config->arguments, "NUM_INPUTS");
Index: /branches/eam_branches/ipp-20130509/pswarp/src/pswarpLoadAstrometry.c
===================================================================
--- /branches/eam_branches/ipp-20130509/pswarp/src/pswarpLoadAstrometry.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/pswarp/src/pswarpLoadAstrometry.c (revision 35747)
@@ -38,4 +38,8 @@
psTrace ("pswarp", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
if (!chip->process || !chip->file_exists) { continue; }
+ pmChip *targetChip = pmFPAviewThisChip(view, target->fpa); // < Chip in the output
+ if (!targetChip || !targetChip->process || !targetChip->file_exists) {
+ continue; // only load astrometry into output chips which exist!
+ }
if (!pmFPAfileRead (astrom, view, config)) {
psError(psErrorCodeLast(), false, "failed READ at CHIP %s", astrom->name);
@@ -109,4 +113,9 @@
if (!chip->process || !chip->file_exists) { continue; }
+ pmChip *targetChip = pmFPAviewThisChip(view, target->fpa); // < Chip in the output
+ if (!targetChip || !targetChip->process || !targetChip->file_exists) {
+ continue; // only load astrometry into output chips which exist!
+ }
+
// we've got the astrom header
pmHDU *hdu = pmHDUFromChip(chip); ///< HDU for source
@@ -116,8 +125,4 @@
return false;
}
-
- // We read from the astrometry source into the target.
- pmChip *targetChip = pmFPAviewThisChip(view, target->fpa); ///< Chip in the output
- if (!targetChip) continue; // only load astrometry into output chips which exist!
if (bilevelAstrometry) {
Index: /branches/eam_branches/ipp-20130509/pswarp/src/pswarpLoopBackground.c
===================================================================
--- /branches/eam_branches/ipp-20130509/pswarp/src/pswarpLoopBackground.c (revision 35746)
+++ /branches/eam_branches/ipp-20130509/pswarp/src/pswarpLoopBackground.c (revision 35747)
@@ -36,5 +36,5 @@
}
- // select the input data sources
+ // select the input data background model
pmFPAfile *output = psMetadataLookupPtr(&status, config->files, "PSWARP.OUTPUT.BKGMODEL");
if (!output) {
Index: /branches/eam_branches/ipp-20130509/tools/cfitsio_tools/Makefile
===================================================================
--- /branches/eam_branches/ipp-20130509/tools/cfitsio_tools/Makefile (revision 35747)
+++ /branches/eam_branches/ipp-20130509/tools/cfitsio_tools/Makefile (revision 35747)
@@ -0,0 +1,2 @@
+immath3d : immath3d.c
+ gcc -o immath3d immath3d.c -Wall -pedantic -Wunreachable-code -march=native -msse2 -mmmx -mfpmath=sse -falign-loops -falign-functions -maccumulate-outgoing-args -time -ggdb3 -O3 -lcfitsio -lm
Index: /branches/eam_branches/ipp-20130509/tools/cfitsio_tools/immath3d.c
===================================================================
--- /branches/eam_branches/ipp-20130509/tools/cfitsio_tools/immath3d.c (revision 35747)
+++ /branches/eam_branches/ipp-20130509/tools/cfitsio_tools/immath3d.c (revision 35747)
@@ -0,0 +1,193 @@
+#define __USE_GNU
+#define _GNU_SOURCE
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+
+
+int main (int argc, char *argv[]) {
+ fitsfile *A;
+ fitsfile *B;
+ fitsfile *O;
+
+ int status = 0;
+
+ int c = 0;
+ int opt_index;
+
+ int operation = 0;
+
+ static struct option long_options[] = {
+ {"help", no_argument, 0, 'h'},
+ {"add", no_argument, 0, 'A'},
+ {"subtract", no_argument, 0, 'S'},
+ {"multiply", no_argument, 0, 'M'},
+ {"divide", no_argument, 0, 'D'},
+ {0,0,0,0}
+ };
+
+ while ((c = getopt_long(argc,argv,"hASMD",
+ long_options,&opt_index)) != -1) {
+ switch (c) {
+ case 'h':
+ fprintf(stderr,"immath3d [options] imageA imageB outimage\n");
+ fprintf(stderr," -A, --add (%d)\n",operation & 0x01);
+ fprintf(stderr," -S, --subtract (%d)\n",operation & 0x02);
+ fprintf(stderr," -M, --multiply (%d)\n",operation & 0x04);
+ fprintf(stderr," -D, --divide (%d)\n",operation & 0x08);
+ exit(1);
+ case 'A':
+ operation = 0x01;
+ break;
+ case 'S':
+ operation = 0x02;
+ break;
+ case 'M':
+ operation = 0x04;
+ break;
+ case 'D':
+ operation = 0x08;
+ break;
+ case '?':
+ if (isprint(optopt)) {
+ fprintf(stderr, "Unknown option `-%c'.\n",optopt);
+ }
+ else {
+ fprintf(stderr, "Unknown option character `\\x%x'.\n",optopt);
+ }
+ break;
+ default:
+ fprintf(stderr, "Unknown option `-%c'.\n",optopt);
+ break;
+ }
+ }
+
+ fprintf(stderr,"%s %s (0x%x) %s\n",argv[optind],argv[optind + 1],operation,argv[optind + 2]);
+
+#define FRE fits_report_error(stderr,status); if (status != 0) { fprintf(stderr, " AT %s:%d\n",__FILE__,__LINE__); } ; status = 0;
+
+ // Open everything up
+ fits_open_file(&A,argv[optind],READONLY,&status); FRE;
+ fits_open_file(&B,argv[optind + 1], READONLY,&status); FRE;
+ fits_create_file(&O,argv[optind + 2], &status); FRE;
+
+ // Copy the structure of A to O
+ fits_copy_file(A,O,1,1,1,&status); FRE;
+
+ // Iterate over the HDUs to perform this operation on all extensions
+ int hdunum;
+ fits_get_num_hdus(A,&hdunum,&status); FRE;
+ int i;
+ for (i = 1; i < hdunum; i++) {
+ printf("%d\n",i);
+ int j;
+ // Get everyone on the same page.
+ int type;
+ fits_movabs_hdu(A,i,&type,&status); FRE;
+ fits_movabs_hdu(B,i,&type,&status); FRE;
+ fits_movabs_hdu(O,i,&type,&status); FRE;
+
+ if (type != IMAGE_HDU) { continue; } // Skip tables, I guess.
+
+ // Get dimension
+ int dimA;
+ int dimB;
+ int dimO;
+ fits_get_img_dim(A,&dimA,&status); FRE;
+ fits_get_img_dim(B,&dimB,&status); FRE;
+ fits_get_img_dim(O,&dimO,&status); FRE;
+ if (dimA == 0) {
+ continue;
+ }
+
+ // Get size
+ long *fpixelA = malloc(dimA * sizeof(long));
+ long *fpixelB = malloc(dimB * sizeof(long));
+ long *fpixelO = malloc(dimO * sizeof(long));
+ long *lpixelA = malloc(dimA * sizeof(long));
+ long *lpixelB = malloc(dimB * sizeof(long));
+ long *lpixelO = malloc(dimO * sizeof(long));
+ fits_get_img_size(A,dimA,lpixelA,&status); FRE;
+ fits_get_img_size(B,dimB,lpixelB,&status); FRE;
+ fits_get_img_size(O,dimO,lpixelO,&status); FRE;
+
+ long nA = lpixelA[0]; fpixelA[0] = 1;
+ long nB = lpixelB[0]; fpixelB[0] = 1;
+ long nO = lpixelO[0]; fpixelO[0] = 1;
+ for (j = 1; j < dimA; j++) { nA *= lpixelA[j]; fpixelA[j] = 1;}
+ for (j = 1; j < dimB; j++) { nB *= lpixelB[j]; fpixelB[j] = 1;}
+ for (j = 1; j < dimO; j++) { nO *= lpixelO[j]; fpixelO[j] = 1;}
+
+ printf(" %d %d %d\n",dimA,dimB,dimO);
+
+ // Get data
+ double *dataA = malloc(sizeof(double) * nA);
+ double *dataB = malloc(sizeof(double) * nB);
+ double *dataO = malloc(sizeof(double) * nO);
+
+ fits_read_pix(A,TDOUBLE,fpixelA,nA,
+ NULL,dataA,NULL,&status); FRE;
+ fits_read_pix(B,TDOUBLE,fpixelB,nB,
+ NULL,dataB,NULL,&status); FRE;
+
+ // Do operation
+ for (j = 0; j < nA; j++) {
+ dataO[j] = 0.0;
+ switch (operation) {
+ case 0x01:
+ dataO[j] = dataA[j] + dataB[j];
+ break;
+ case 0x02:
+ dataO[j] = dataA[j] - dataB[j];
+ break;
+ case 0x04:
+ dataO[j] = dataA[j] * dataB[j];
+ break;
+ case 0x08:
+ dataO[j] = dataA[j] / dataB[j];
+ break;
+ }
+ }
+
+ // Save this output
+ fits_write_pix(O,TDOUBLE,fpixelO,nO,dataO,&status); FRE;
+
+ // Cleanup
+ free(fpixelA);
+ free(fpixelB);
+ free(fpixelO);
+ free(lpixelA);
+ free(lpixelB);
+ free(lpixelO);
+
+ free(dataA);
+ free(dataB);
+ free(dataO);
+
+ }
+
+ fits_close_file(A,&status); FRE;
+ fits_close_file(B,&status); FRE;
+ fits_close_file(O,&status); FRE;
+ return(0);
+}
+
Index: /branches/eam_branches/ipp-20130509/tools/runchipimfile.pl
===================================================================
--- /branches/eam_branches/ipp-20130509/tools/runchipimfile.pl (revision 35746)
+++ /branches/eam_branches/ipp-20130509/tools/runchipimfile.pl (revision 35747)
@@ -77,4 +77,5 @@
if ($outdir) {
+ $outdir = $ENV{PWD} if $outdir eq '.';
my $base = basename($path_base);
$path_base = "$outdir/$base";
Index: /branches/eam_branches/ipp-20130509/tools/smf2ct.pl
===================================================================
--- /branches/eam_branches/ipp-20130509/tools/smf2ct.pl (revision 35747)
+++ /branches/eam_branches/ipp-20130509/tools/smf2ct.pl (revision 35747)
@@ -0,0 +1,146 @@
+#! /usr/bin/env perl
+
+use warnings;
+use strict;
+
+
+my $smf = shift(@ARGV);
+my $ra = shift(@ARGV);
+my $dec = shift(@ARGV);
+
+# Test case
+#$smf = '/data/ipp019.0/nebulous/7e/aa/3519801261.gpc1:ThreePi.nt:2013:06:13:o6456g0187o.619280:o6456g0187o.619280.cm.821906.smf';
+#$ra = 270.7396666667;
+#$dec = 7.0036111111;
+
+unless(defined($smf)&&
+ defined($ra)&&
+ defined($dec)) {
+ die "smf2ct.pl SMF RA DEC";
+}
+
+# ppCoord requires a file, not command line positions
+open(RD,">/tmp/rd.$$");
+print RD "$ra $dec\n";
+close(RD);
+
+# make call to ppCoord to convert RD to XYchip
+open(ppC,"ppCoord -astrom $smf -radec /tmp/rd.$$ |");
+while() {
+ chomp;
+ my ($r,$d,undef,$x,$y,$chip) = split /\s+/;
+
+ my (undef,undef,$otaU,$otaV) = split //,$chip; # ;/;
+
+ my ($CTchip,$CTx,$CTy) = ("NO_CT",-1,-1);
+
+ {
+ # Search SMF
+ my @source_match = ();
+ my $match_R = 1.0;
+# open(FD,"fdump $smf ${chip}.psf |");
+ open(FD,"ftable $smf -n ${chip}.psf |");
+ while () {
+ chomp;
+ if ($_ =~ /IPP/) { next; }
+ my @row = split /\s+/;
+ # fragile
+ my ($smf_x,$smf_y,$smf_ra,$smf_dec,$smf_flag,$smf_inst_mag,$smf_inst_mag_sig) =
+ ($row[1],$row[2],$row[16],$row[17],
+ $row[44],$row[7],$row[8]);
+ if (sqrt(($smf_x - $x)**2 + ($smf_y - $y)**2) < $match_R) {
+ $match_R = sqrt(($smf_x - $x)**2 + ($smf_y - $y)**2);
+ @source_match = ($smf_x,$smf_y,$smf_ra,$smf_dec,$smf_flag,$smf_inst_mag,$smf_inst_mag_sig);
+ }
+ }
+ close(FD);
+ printf("%.10f %.10f -> %s %f %f\n",
+ $ra,$dec,$chip,$x,$y);
+ printf(" %f %f (%f,%f) %x %f %f\n",
+ @source_match);
+ if ($source_match[4] & 1073741824) {
+ printf(" This source flagged as crosstalk.\n");
+ }
+ }
+
+ # Apply known rules
+ if ($otaU == 2) {
+ if (OTA_cell_col($x) == 3) {
+ ($CTchip,$CTx,$CTy) = ("XY3${otaV}",$x,$y);
+ }
+ if (OTA_cell_col($x) == 5) {
+ ($CTchip,$CTx,$CTy) = ($chip,$x + 590 + 18,$y);
+ }
+ if (OTA_cell_col($x) == 6) {
+ ($CTchip,$CTx,$CTy) = ($chip,$x - 590 - 18,$y);
+ }
+ }
+ if ($otaU == 3) {
+ if (OTA_cell_col($x) == 3) {
+ ($CTchip,$CTx,$CTy) = ("XY2${otaV}",$x,$y);
+ }
+ }
+ if ($otaU == 4) {
+ if (OTA_cell_col($x) == 3) {
+ ($CTchip,$CTx,$CTy) = ("XY5${otaV}",$x,$y);
+ }
+ }
+ if ($otaU == 5) {
+ if (OTA_cell_col($x) == 3) {
+ ($CTchip,$CTx,$CTy) = ("XY4${otaV}",$x,$y);
+ }
+ if (OTA_cell_col($x) == 5) {
+ ($CTchip,$CTx,$CTy) = ($chip,$x + 590 + 18,$y);
+ }
+ if (OTA_cell_col($x) == 6) {
+ ($CTchip,$CTx,$CTy) = ($chip,$x - 590 - 18,$y);
+ }
+ }
+
+ if ($CTchip eq "NO_CT") {
+ printf(" %s %f %f (%f,%f) %x %f %f\n",
+ $CTchip,$CTx,$CTy,-1,-1,0,-1,-1);
+ }
+ else {
+ # Search SMF for CT source
+ my @source_match = ();
+ my $match_R = 1.0;
+# open(FD,"fdump $smf ${CTchip}.psf |");
+ open(FD,"ftable $smf -n ${CTchip}.psf |");
+ while () {
+ chomp;
+ if ($_ =~ /IPP/) { next; }
+ my @row = split /\s+/;
+ # fragile
+ my ($smf_x,$smf_y,$smf_ra,$smf_dec,$smf_flag,$smf_inst_mag,$smf_inst_mag_sig) =
+ ($row[1],$row[2],$row[16],$row[17],
+ $row[44],$row[7],$row[8]);
+ if (sqrt(($smf_x - $CTx)**2 + ($smf_y - $CTy)**2) < $match_R) {
+ $match_R = sqrt(($smf_x - $x)**2 + ($smf_y - $y)**2);
+ @source_match = ($smf_x,$smf_y,$smf_ra,$smf_dec,$smf_flag,$smf_inst_mag,$smf_inst_mag_sig);
+ }
+ }
+ close(FD);
+ printf("CT_SOURCE -> %s %f %f\n",
+ $CTchip,$CTx,$CTy);
+ printf(" %f %f (%f,%f) %x %f %f\n",
+ @source_match);
+ }
+}
+system("rm /tmp/rd.$$");
+
+sub OTA_cell_col {
+ # cells are 590x598, with 18x12 gaps between
+ my $inX = shift;
+ if ($inX < 590) { return (0); } # col 0
+ my $i;
+ for ($i = 1; $i <= 7; $i++) { # col 1-7
+ if (($inX > ($i * (590 + 18)))&&
+ ($inX < (($i + 1) * (590 + 18)))) {
+ return($i);
+ }
+ }
+}
+
+
+