Index: /branches/eam_branches/ipp-20121130/Nebulous-Server/bin/neb-host
===================================================================
--- /branches/eam_branches/ipp-20121130/Nebulous-Server/bin/neb-host (revision 34838)
+++ /branches/eam_branches/ipp-20121130/Nebulous-Server/bin/neb-host (revision 34839)
@@ -141,6 +141,6 @@
$dbh->commit;
-my $format = "%-15s %-15s %-7s %-8s %-9s %-5s %s\n";
-my @columns = qw(host name mounted allocate available xattr note);
+my $format = "%-15s %-15s %-7s %-8s %-9s %-5s %15s %s\n";
+my @columns = qw(host name mounted allocate available xattr last_modified note);
printf($format, @columns);
Index: /branches/eam_branches/ipp-20121130/Nebulous-Server/bin/neb-voladm
===================================================================
--- /branches/eam_branches/ipp-20121130/Nebulous-Server/bin/neb-voladm (revision 34838)
+++ /branches/eam_branches/ipp-20121130/Nebulous-Server/bin/neb-voladm (revision 34839)
@@ -68,5 +68,5 @@
if (defined $allocate and $allocate !~ m/^[01]$/)
or (defined $available and $available !~ m/^[01]$/)
- or (defined $xattr and $xattr !~ m/^[012]$/);
+ or (defined $xattr and $xattr !~ m/^[0123]$/);
}
Index: /branches/eam_branches/ipp-20121130/Nebulous-Server/lib/Nebulous/Server/SQL.pm
===================================================================
--- /branches/eam_branches/ipp-20121130/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 34838)
+++ /branches/eam_branches/ipp-20121130/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 34839)
@@ -389,4 +389,5 @@
mountedvol.vol_id IS NOT NULL as mounted,
v.cab_id,
+ v.last_modified,
v.note
FROM volume AS v
@@ -694,4 +695,5 @@
cab_id INT,
note VARCHAR(255),
+ last_modified TIMESTAMP NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, # SC: Added by Haydn with ALTER TABLE(?) on 2012-12-03
PRIMARY KEY(vol_id),
UNIQUE KEY(name),
Index: /branches/eam_branches/ipp-20121130/ippScripts/Build.PL
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/Build.PL (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/Build.PL (revision 34839)
@@ -128,4 +128,5 @@
scripts/listvideocells.pl
scripts/skycalibration.pl
+ scripts/regenerate_background.pl
)],
dist_abstract => 'Scripts for running the Pan-STARRS IPP',
Index: /branches/eam_branches/ipp-20121130/ippScripts/MANIFEST
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/MANIFEST (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/MANIFEST (revision 34839)
@@ -46,3 +46,4 @@
scripts/skycell_jpeg.pl
scripts/lap_science.pl
+scripts/regenerate_background.pl
t/00_distribution.t
Index: /branches/eam_branches/ipp-20121130/ippScripts/scripts/chip_imfile.pl
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/scripts/chip_imfile.pl (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/scripts/chip_imfile.pl (revision 34839)
@@ -150,4 +150,5 @@
$outputStats = prepare_output("PPIMAGE.STATS", $outroot, $class_id, 1);
my $do_binned_images = 1;
+
if ($run_state eq 'new') {
# prepare the files that are only created for a new run
@@ -586,5 +587,7 @@
# check whether psphot outputs should be regenerated.
-# Whether we need to or not is a somewhat complicated question.
+# Whether we need to or not was a somewhat complicated question. Since we clean cmfs now we only rerun
+# photometry if the psf file has gone missing since it is needed to run warps
+
sub rerun_photometry
{
@@ -594,4 +597,6 @@
or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+ my $update_sources_if_gone = 0; # set this to regenerate sources if gone.
+ # We no longer do this as of 2012-12
my $make_sources = 0;
my $sources_available = 0;
@@ -599,15 +604,17 @@
$sources_available = 1;
} else {
- carp "WARNING: photometry sources file $outputSources is not available";
- my $gone;
- if (storage_object_exists($outputSources, \$gone)) {
- # check whether the file is permanantely or temporarily gone
- if ($gone) {
- carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
+ if ($update_sources_if_gone) {
+ carp "WARNING: photometry sources file $outputSources is not available";
+ my $gone;
+ if (storage_object_exists($outputSources, \$gone)) {
+ # check whether the file is permanantely or temporarily gone
+ if ($gone) {
+ carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
+ $make_sources = 1;
+ }
+ } else {
+ # storage object must have been deleted
$make_sources = 1;
}
- } else {
- # storage object must have been deleted
- $make_sources = 1;
}
}
@@ -616,5 +623,7 @@
my $psf_available = 0;
my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE", $outroot, $class_id)
- or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+ or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE",
+ $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+
if ($ipprc->file_exists($outputPsf)) {
$psf_available = 1;
@@ -638,10 +647,12 @@
}
- # if either of the files are gone rerun photometry unless the other file is temporarily not available
-
- if (!$sources_available && !$make_sources) {
- # destreak will die if the sources is not available
- &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
- }
+ if ($update_sources_if_gone) {
+ if (!$sources_available && !$make_sources) {
+ # destreak will die if the sources is not available
+ # but magic is dead....
+ &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+ }
+ }
+
if (!$psf_available && !$make_psf) {
# warp updates need the psf file
Index: /branches/eam_branches/ipp-20121130/ippScripts/scripts/detrend_process_imfile.pl
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/scripts/detrend_process_imfile.pl (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/scripts/detrend_process_imfile.pl (revision 34839)
@@ -27,4 +27,6 @@
my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1);
+
if ($missing_tools) {
warn("Can't find required tools.");
@@ -124,4 +126,11 @@
# Run ppImage
unless ($no_op) {
+ my $repair_cmd = "$nebrepair $input_uri";
+ my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+ unless ($repair_success) {
+ &my_die("Unable to attempt repair: $input_uri $repair_error_code", $det_id, $exp_id, $class_id $PS_EXIT_SYS_ERROR);
+ }
+
+
my $command = "$ppImage -file $input_uri $outroot";
$command .= " -recipe PPIMAGE $ppimage_recipe";
Index: /branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_cleanup.pl (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_cleanup.pl (revision 34839)
@@ -18,5 +18,12 @@
# Parse the command-line arguments
-my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile);
+my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile, $check_all);
+my $very_verbose = 0;
+
+# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
+#my $ds_done = 0;
+# magic is dead
+my $ds_done = 1;
+
GetOptions('stage=s' => \$stage, # which analysis stage to clean?
'camera|i=s' => \$camera, # user-supplied camera name
@@ -24,4 +31,5 @@
'mode|m=s' => \$mode, # cleanup mode (clean / purge)
'path_base=s' => \$path_base, # basename for files
+ 'check-all' => \$check_all, # if set clean all chips regardless of data_state
'dbname|d=s' => \$dbname, # Database name
'verbose' => \$verbose, # Print to stdout
@@ -56,4 +64,7 @@
$ipprc->redirect_output($logfile) or
&my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile;
+
+
+my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n';
# set this to 1 to enable checking for files on dead nodes
@@ -91,4 +102,5 @@
# this stage uses 'chiptool'
my $chiptool = can_run('chiptool') or die "Can't find chiptool";
+ my $censorObjects = can_run('censorObjects') or die "Can't find censorObjects";
# Get list of component imfiles
@@ -96,6 +108,9 @@
my $imfiles; # Array of component files
my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run
+ $command .= ' -all' if ($check_all);
$command .= " -dbname $dbname" if defined $dbname;
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf )
+ = run(command => $command, verbose => $very_verbose);
unless ($success) {
$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -114,5 +129,5 @@
unless ($success) {
$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+ &my_die("Unable to perform chiptool -processedimfile: $error_code", "chip", $stage_id, $error_code);
}
exit 0;
@@ -123,28 +138,104 @@
&my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
+ my $numchips = scalar @$imfiles;
+ print "Found $numchips to clean\n";
+
+ my $clean_sources = 0;
+ if ((scalar @$imfiles > 0) and ($mode eq 'goto_cleaned')) {
+ # go and find the smf file(s) for the associated camRun and check the status of the file
+ # if a good one is found we have the sources for this chipRun and thus can clean the cmfs
+ my $command = "$chiptool -listrun -chip_id $stage_id";
+ $command .= " -dbname $dbname" if defined $dbname;
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform chiptool -listrun: $error_code", "chip", $stage_id, $error_code);
+ }
+ my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or
+ &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
+ my $good_smf = 0;
+ foreach my $entry (@$entries) {
+ my $camRun_state = $entry->{camRun_state};
+ next if $camRun_state ne 'full';
+ my $cam_id = $entry->{cam_id};
+ if (!$cam_id) {
+ carp('no cam_id for listrun entry');
+ next;
+ }
+ my $cam_path_base = $entry->{cam_path_base};
+ if ( !defined $cam_path_base ) {
+ carp("no path_base for $cam_id\n");
+ next;
+ }
+
+ # XXX: This assumes that the filerules are filerules-split
+ my $smf = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
+ if (!$ipprc->file_exists($smf)) {
+ carp("smf for $cam_path_base not found");
+ next;
+ }
+ # we run the program censorObjects in the check mode
+ # If this program succeeds the smf is a valid fits file and each of the
+ # extensions was succesfully read.
+ # XXX: create a new program outside of magic that performs this check
+
+ my $command = "$censorObjects -checkinputonly -file $smf";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $very_verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ print STDERR "censorObjects failed:\n";
+ print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf;
+ print STDERR "\nSTDERR:\n" . join "", @$stderr_buf;
+ &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code);
+ }
+ $good_smf++;
+ }
+ if ($good_smf) {
+ # we have a good one so we can clean the sources
+ $clean_sources = 1;
+ print "Found $good_smf good smf files will clean sources\n";
+ } else {
+ print "Unable to find good smf file will NOT clean sources\n";
+ }
+ }
+
# loop over all of the imfiles, determine the path_base and class_id for each
+ my $num_errors = 0;
foreach my $imfile (@$imfiles) {
my $class_id = $imfile->{class_id};
my $path_base = $imfile->{path_base};
+ my $data_state = $imfile->{data_state};
my $status = 1;
$status = 0 unless defined $path_base and $path_base ne "NULL";
- my $poor_quality = $imfile->{quality} > 0;
+ my $quality = $imfile->{quality};
+ my $good_quality = ($quality == 0);
+
+ print "Starting cleanup for $class_id\n";
# don't clean up unless the data needed to update is available
# modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
# goto_scrubbed now requires the config file to not exist.
- if ($status and !$poor_quality) {
+ if ($status and $good_quality) {
if ($mode eq "goto_cleaned") {
my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
unless ($ipprc->file_exists($config_file)) {
+ my $fault = $imfile->{fault};
+
if (file_gone($config_file)) {
- print STDERR "forcing cleanup for chipRun $stage_id $class_id "
+ # config file was lost. Clean up. If the chip is ever updated a new config
+ # file will be created
+ print STDERR "forcing cleanup chip $stage_id $class_id fault: $fault quality: $quality"
. " because config file ($config_file) is gone\n";
+ } elsif ($fault == 0 and $quality == 0) {
+ print STDERR "skipping cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
+ . " because config file ($config_file) is missing\n";
+ $status = 0;
} else {
- print STDERR "skipping cleanup for chipRun $stage_id $class_id "
- . " because config file ($config_file) is missing\n";
- $status = 0;
+ # config file is missing but this is a bad chip anyways so clean it
+ print STDERR "cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
+ . " even though config file ($config_file) is missing\n";
}
}
@@ -154,5 +245,5 @@
if ($ipprc->file_exists($config_file)) {
- print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
+ print STDERR "skipping scrubbed for chip $stage_id $class_id "
. " because config file ($config_file) is present\n";
$status = 0;
@@ -165,21 +256,30 @@
my @files = ();
- # delete the temporary image datafiles
-# addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
-# addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
-# addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0);
+ if ($clean_sources) {
+ addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
+ addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+ }
if ($mode eq "goto_purged") {
# additional files to remove for 'purge' mode
+ if (!$clean_sources) {
+ # these weren't added above but we do want to clean it
+ addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
+ addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+ }
+
+ # background model is needed to build stack background images so we do not remove it
+ # addFilename (\@files, "PSPHOT.BACKMDL", $path_base, $class_id);
+
+ addFilename (\@files, "PSPHOT.PSF.SAVE", $path_base, $class_id);
addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
- addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); # clean?
- addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); # clean?
+ addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
- addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); #clean?
+ addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
}
@@ -188,29 +288,42 @@
$status = &delete_files (\@files);
}
+ bzip2_file("LOG.IMFILE", $path_base, $class_id);
+ bzip2_file("LOG.IMFILE.UPDATE", $path_base, $class_id);
if ($status) {
+ my $update_chip = 1;
my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
if ($mode eq "goto_purged") {
$command .= " -topurgedimfile";
+ if ($data_state eq 'purged') {
+ $update_chip = 0;
+ }
}
elsif ($mode eq "goto_cleaned") {
$command .= " -tocleanedimfile";
+ if ($data_state eq 'cleaned') {
+ $update_chip = 0;
+ }
}
elsif ($mode eq "goto_scrubbed") {
$command .= " -toscrubbedimfile";
- }
-
- $command .= " -dbname $dbname" if defined $dbname;
-
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => $verbose);
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
- }
-
- set_destreak_goto_cleaned();
-
+ if ($data_state eq 'scrubbed') {
+ $update_chip = 0;
+ }
+ }
+
+ if ($update_chip) {
+ $command .= " -dbname $dbname" if defined $dbname;
+
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+ }
+ set_destreak_goto_cleaned();
+ }
} else {
+ $num_errors++;
# if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
@@ -218,6 +331,4 @@
$command .= " -dbname $dbname" if defined $dbname;
-if (0) {
- # XXX Don't set components to error cleaned anymore
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
run(command => $command, verbose => $verbose);
@@ -226,5 +337,4 @@
&my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
}
-}
# We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
@@ -241,4 +351,7 @@
}
}
+ print "Cleanup completed for chip_id $stage_id.";
+ print " num_errors: $num_errors" if $num_errors;
+ print "\n";
exit 0;
}
@@ -350,7 +463,8 @@
my $skyfiles; # Array of component files
my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run
+ $command .= ' -all' if $check_all;
$command .= " -dbname $dbname" if defined $dbname;
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => $verbose);
+ run(command => $command, verbose => $very_verbose);
unless ($success) {
$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -378,8 +492,13 @@
&my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
+ my $numskycells = scalar @$skyfiles;
+ print "Found $numskycells to clean\n";
+
my @files = ();
+ my $num_errors = 0;
foreach my $skyfile (@$skyfiles) {
my $path_base = $skyfile->{path_base};
my $skycell_id = $skyfile->{skycell_id};
+ my $data_state = $skyfile->{data_state};
my $status = 1;
@@ -391,11 +510,17 @@
unless ($ipprc->file_exists($config_file)) {
+ my $fault = $skyfile->{fault};
+ my $quality = $skyfile->{quality};
if (file_gone($config_file)) {
- print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" .
- " because config file is gone\n";
+ print STDERR "forcing cleanup warp $stage_id $skycell_id fault: $fault quality: $quality"
+ . " because config file ($config_file) is gone\n";
+ } elsif ($fault == 0 and $quality == 0) {
+ print STDERR "skipping cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
+ . " because config file ($config_file) is missing\n";
+ $status = 0;
} else {
- print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
- " because config file is missing\n";
- $status = 0;
+ # config file is missing but this is a bad warp anyways so clean it
+ print STDERR "cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
+ . " even though config file ($config_file) is missing\n";
}
}
@@ -413,5 +538,6 @@
if ($status) {
- if ($skyfile->{quality} != 8007) {
+ # XXX: what is special about quality == 8007?
+ if ($skyfile->{quality} != 8007 || $check_all) {
my @files = ();
@@ -420,5 +546,7 @@
addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
-# addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
+ # these are rebuilt during update so we can delete them here
+ addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
+ addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id );
if ($mode eq "goto_purged") {
# additional files to remove for 'purge' mode
@@ -426,30 +554,58 @@
addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
- # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);
-
- # XXX: do we want to delete these?
+ addFilename(\@files, "SKYCELL.STATS.UPDATE", $path_base, $skycell_id );
+ addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
+
+ # XXX: do we want to delete these? trace file is empty
# addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
- # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
- }
- # actual command to delete the files
+ }
+ # actual command to delete the files
$status = &delete_files (\@files);
}
}
+ bzip2_file("LOG.EXP", $path_base, $skycell_id);
+ bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
if ($status) {
+ my $update_skyfile = 1;
my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
if ($mode eq "goto_purged") {
$command .= " -topurgedskyfile";
+ if ($data_state eq 'purged') {
+ $update_skyfile = 0;
+ }
}
elsif ($mode eq "goto_cleaned") {
$command .= " -tocleanedskyfile";
+ if ($data_state eq 'cleaned') {
+ $update_skyfile = 0;
+ }
}
elsif ($mode eq "goto_scrubbed") {
$command .= " -toscrubbedskyfile";
- }
- $command .= " -dbname $dbname" if defined $dbname;
-
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => $verbose);
+ if ($data_state eq 'scrubbed') {
+ $update_skyfile = 0;
+ }
+ }
+ $command .= " -dbname $dbname" if defined $dbname;
+
+ if ($update_skyfile) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+ }
+
+ set_destreak_goto_cleaned();
+ }
+
+ } else {
+ $num_errors++;
+ my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
+ $command .= " -dbname $dbname" if defined $dbname;
+
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
unless ($success) {
$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -457,11 +613,9 @@
}
- set_destreak_goto_cleaned();
-
- } else {
- my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
- $command .= " -dbname $dbname" if defined $dbname;
-
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
+ $command = "$warptool -warp_id $stage_id -updaterun -set_state $error_state";
+ $command .= " -dbname $dbname" if defined $dbname;
+
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
run(command => $command, verbose => $verbose);
unless ($success) {
@@ -469,8 +623,9 @@
&my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
}
-
- # exit $PS_EXIT_UNKNOWN_ERROR;
- }
- }
+ }
+ }
+ print "Cleanup completed for warp_id $stage_id.";
+ print " num_errors: $num_errors" if $num_errors;
+ print "\n";
exit 0;
}
@@ -603,5 +758,4 @@
&my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
}
-# exit $PS_EXIT_UNKNOWN_ERROR;
}
}
@@ -619,6 +773,7 @@
my $skyfiles; # Array reference of component files
my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run
+ $command .= ' -all' if $check_all;
$command .= " -dbname $dbname" if defined $dbname;
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
unless ($success) {
$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -646,8 +801,10 @@
&my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
+ my $num_errors = 0;
my @files = ();
foreach my $skyfile (@{ $skyfiles }) {
my $path_base = $skyfile->{path_base};
my $skycell_id = $skyfile->{skycell_id};
+ my $data_state = $skyfile->{data_state};
my $status = 1;
@@ -661,4 +818,5 @@
my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
+ if (0) {
unless ($ipprc->file_exists($config_file)) {
if (file_gone($config_file)) {
@@ -672,4 +830,21 @@
}
}
+ unless ($ipprc->file_exists($config_file)) {
+ my $fault = $skyfile->{fault};
+ my $quality = $skyfile->{quality};
+ if (file_gone($config_file)) {
+ print STDERR "forcing cleanup diff $stage_id $skycell_id fault: $fault quality: $quality"
+ . " because config file ($config_file) is gone\n";
+ } elsif ($fault == 0 and $quality == 0) {
+ print STDERR "skipping cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
+ . " because config file ($config_file) is missing\n";
+ $status = 0;
+ } else {
+ # config file is missing but this is a bad diff anyways so clean it
+ print STDERR "cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
+ . " even though config file ($config_file) is missing\n";
+ }
+ }
+ }
elsif ($mode eq "goto_scrubbed") {
my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
@@ -712,21 +887,49 @@
}
-# print STDERR "MY FILES: @files\n";
$status = &delete_files(\@files);
}
-# print STDERR "MY STATUS: $status\n";
+
+ bzip2_file("LOG.EXP", $path_base, $skycell_id);
+ bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
+
if ($status) {
my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
+ my $update_skyfile = 1;
if ($mode eq "goto_purged") {
$command .= " -topurgedskyfile";
+ if ($data_state eq 'purged') {
+ $update_skyfile = 0;
+ }
}
elsif ($mode eq "goto_cleaned") {
$command .= " -tocleanedskyfile";
+ if ($data_state eq 'cleaned') {
+ $update_skyfile = 0;
+ }
}
elsif ($mode eq "goto_scrubbed") {
$command .= " -toscrubbedskyfile";
- }
-
+ if ($data_state eq 'scrubbed') {
+ $update_skyfile = 0;
+ }
+ }
+
+ $command .= " -dbname $dbname" if defined $dbname;
+
+ if ($update_skyfile) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
+ }
+ set_destreak_goto_cleaned();
+ }
+
+
+ } else {
+ $num_errors++;
+ my $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $skycell_id -set_state $error_state";
$command .= " -dbname $dbname" if defined $dbname;
@@ -738,12 +941,9 @@
}
- set_destreak_goto_cleaned();
-
- } else {
- my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
-
- $command .= " -dbname $dbname" if defined $dbname;
-
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
+
+ $command .= " -dbname $dbname" if defined $dbname;
+
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
run(command => $command, verbose => $verbose);
unless ($success) {
@@ -751,7 +951,9 @@
&my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
}
-# exit $PS_EXIT_UNKNOWN_ERROR;
- }
- }
+ }
+ }
+ print "Cleanup completed for diff_id $stage_id.";
+ print " num_errors: $num_errors" if $num_errors;
+ print "\n";
exit 0;
}
@@ -1917,5 +2119,5 @@
foreach my $file (@$files) {
- print STDERR "unlinking $stage $stage_id $file\n";
+ print STDERR "unlinking $stage $stage_id $file\n" if $very_verbose;
my $error_code = $ipprc->kill_file($file);
@@ -2008,6 +2210,4 @@
}
-# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
-my $ds_done = 0;
sub set_destreak_goto_cleaned {
@@ -2027,4 +2227,37 @@
}
+sub bzip2_file {
+ my $filerule = shift;
+ my $path_base = shift;
+ my $component = shift;
+
+ my $filename = $ipprc->filename($filerule, $path_base, $component);
+ if (!$ipprc->file_exists($filename)) {
+ return 1;
+ }
+ if (my $resolved = $ipprc->file_resolve($filename)) {
+ my $bzip2_filename = $filename . '.bz2';
+ if ($ipprc->file_exists($bzip2_filename)) {
+ $ipprc->kill_file($bzip2_filename);
+ }
+ my $bzip2_file = $ipprc->file_create($bzip2_filename);
+ my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file;
+
+ my $command = "$bzip2 < $resolved > $bzip2_file";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $very_verbose);
+ if ($success) {
+ # success delete the original file
+ my $error_code = $ipprc->kill_file($filename);
+ } else {
+ # if bzip2 failed. Carry on but don't delete the existing file
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ print STDERR "Failed to bzip2 $filename: $error_code\n";
+ return 0;
+ }
+ }
+ return 1;
+}
+
# XXX we currently do not set the error state in the db on my_die
sub my_die
Index: /branches/eam_branches/ipp-20121130/ippScripts/scripts/regenerate_background.pl
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/scripts/regenerate_background.pl (revision 34839)
+++ /branches/eam_branches/ipp-20121130/ippScripts/scripts/regenerate_background.pl (revision 34839)
@@ -0,0 +1,358 @@
+#! /usr/bin/env perl
+
+# Script to call the appropriate commands to regenerate the background models
+# for stages subsequent to the chip stage.
+
+use warnings;
+use strict;
+use Carp;
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Spec;
+use File::Temp qw( tempfile );
+use PS::IPP::Config 1.01 qw( :standard );
+use Getopt::Long qw (GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+
+
+my $ppConfigDump = can_run('ppConfigDump');
+my $pswarp = can_run('pswarp');
+
+# Parse command line options.
+my ($stage, $camera, $stage_id, $skycell_id, $dbname, $verbose, $logfile, $save_temps);
+GetOptions('stage=s' => \$stage, # which analysis stage to generate bkg model
+ 'camera|i=s' => \$camera, # user-supplied camera name
+ 'stage_id=s' => \$stage_id, # id for this stage
+ 'skycell_id=s' => \$skycell_id, # skycell_id
+ 'verbose' => \$verbose, # verbose commands
+ 'logfile=s' => \$logfile, # destination for stdout and stderr
+ 'save_temps' => \$save_temps, # save temporary files.
+ 'dbname=s' => \$dbname # database name
+ ) or pod2usage ( 2 );
+
+pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) [--dbname (dbname)]",
+ -exitval => 2 ) if scalar @ARGV;
+
+pod2usage( -msg => "Required options: --camera (name) --stage (stage) --stage_id (stage_id)",
+ -exitval => 3 ) unless
+ defined $camera and
+ defined $stage and
+ defined $stage_id;
+
+pod2usage( -msg => "Required options: --camera (name) --stage (stage) --stage_id (stage_id) --skycell_id (skycell_id)",
+ -exitval => 3) if (($stage eq 'warp')&&(!defined($skycell_id)));
+
+
+my $ipprc = PS::IPP::Config->new ( $camera ) or
+ &my_die("Unable to set up", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR); # used for path/neb info
+
+# We can only regenerate a background for certain stages.
+unless (($stage eq "warp") || ($stage eq "stack") || ($stage eq "warpstack")) {
+ &my_die("Invalid stage to regenerate", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR);
+}
+
+$ipprc->redirect_output($logfile) or
+ &my_die("Unable to redirect output to logfile", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile;
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # parser for metadata config files
+
+
+## warp stage
+if ($stage eq 'warp') {
+ my $warp_id = $stage_id; # Because I think I used the wrong id in places.
+ &my_die("--stage_id required for stage warp", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if !$stage_id;
+ &my_die("--skycell_id required for stage warp", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if !$skycell_id;
+
+ # Configuration stuff
+ my $warptool = can_run('warptool') or
+ &my_die("Can't find warptool",$stage,$stage_id, $PS_EXIT_UNKNOWN_ERROR);
+
+ my $astromSource; # The astrometry source
+ {
+ my $command = "$ppConfigDump -camera $camera -recipe PSWARP BACKGROUND -dump-recipe PSWARP -";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform ppConfigDump: $error_code",
+ $stage, $stage_id, $error_code);
+ }
+ my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+ &my_die("Unable to parse metadata config doc",
+ $stage, $stage_id, $PS_EXIT_PROG_ERROR);
+ $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
+ }
+
+
+ my $imfiles; # Array of component files
+ my $command = "$warptool -warped -warp_id $stage_id -skycell_id $skycell_id";
+ $command .= " -dbname $dbname " if defined $dbname;
+
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderrr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform warptool: $error_code", $stage, $stage_id, $error_code);
+ }
+
+ if (@$stdout_buf == 0) {
+ # No results
+ }
+
+ my $in_md = $mdcParser->parse(join "", @$stdout_buf) or
+ &my_die("Unable to parse metadata config doc", $stage, $stage_id, $PS_EXIT_PROG_ERROR);
+ $imfiles = parse_md_list($in_md) or
+ &my_die("Unable to parse metadata", $stage, $stage_id, $PS_EXIT_PROG_ERROR);
+
+ foreach my $imfile (@$imfiles) {
+ my $skycell_id = $imfile->{skycell_id};
+ my $path_base = $imfile->{path_base};
+ my $tess_id = $imfile->{tess_id};
+
+ # Parse the results to determine if there are any problems that prevent this from
+ # being processed. I think I might need to think about this a bit more.
+ my $status = 1;
+ $status = 0 unless defined $path_base and $path_base ne "NULL";
+ my $poor_quality = $imfile->{quality} > 0;
+
+# my $bkg_file = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL",$path_base,$skycell_id);
+# my $bkg_exists = $ipprc->file_exists($bkg_file);
+ my $bkg_exists = $imfile->{background_model};
+ if ($bkg_exists) { $status = 0; } # We do not need to remake this.
+
+ # Construct temp files
+ my ($bkgList_file,$bkgList_name) = tempfile("/tmp/bkgreg.warp.bkg.${warp_id}.${skycell_id}.XXXX",
+ UNLINK => !$save_temps);
+ my ($astromList_file,$astromList_name) = tempfile("/tmp/bkgreg.warp.astrom.${warp_id}.${skycell_id}.XXXX",
+ UNLINK => !$save_temps);
+ my $chipFiles;
+ {
+ my $inputs_command = "$warptool -scmap -warp_id $warp_id ";
+ $inputs_command .= " -skycell_id $skycell_id ";
+ $inputs_command .= " -dbname $dbname " if defined $dbname;
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run (command => $inputs_command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform warptool -scmap: $error_code",
+ $skycell_id,$warp_id,$PS_EXIT_SYS_ERROR);
+ }
+ my $md = $mdcParser->parse(join "", @$stdout_buf) or
+ &my_die("Unable to parse warptool -scmap: $error_code",
+ $skycell_id,$warp_id,$PS_EXIT_SYS_ERROR);
+ $chipFiles = parse_md_list($md) or
+ &my_die("Unable to parse warptool -scmap: $error_code",
+ $skycell_id,$warp_id,$PS_EXIT_SYS_ERROR);
+ }
+ my $wrote_astrom = 0;
+ foreach my $chipFile (@$chipFiles) {
+ my $bkgModel = $ipprc->filename("PSPHOT.BACKMDL",
+ $chipFile->{chip_path_base},
+ $chipFile->{class_id});
+ my $astrom = $ipprc->filename($astromSource,$chipFile->{cam_path_base});
+ print $bkgList_file "$bkgModel\n";
+ if (!$wrote_astrom) {
+ print $astromList_file "$astrom\n";
+ $wrote_astrom = 1;
+ }
+ }
+ close ($bkgList_file);
+ close ($astromList_file);
+
+ # Construct skyfile if needed
+ my $skyCell_name = $ipprc->filename("SKYCELL.TEMPLATE",$path_base,$skycell_id);
+ my $skyCell_exists = $ipprc->file_exists($skyCell_name);
+ if (!$skyCell_exists) { # Generate it
+ my $skyFile = prepare_output("SKYCELL.TEMPLATE", $path_base, $skycell_id, 1);
+ $ipprc->skycell_file( $tess_id, $skycell_id, $skyFile, $verbose ) or
+ &my_die("Unable to generate template skycell",$skycell_id,$warp_id,$PS_EXIT_SYS_ERROR);
+ }
+ my $bkgOut_log = prepare_output("LOG.EXP",$path_base . ".BKG_REG",$skycell_id,1);
+
+ # Construct pswarp command
+ my $pswarp_command = "$pswarp ";
+ $pswarp_command .= " -list $bkgList_name -astromlist $astromList_name -bkglist $bkgList_name ";
+ $pswarp_command .= " ${path_base} ";
+ $pswarp_command .= " $skyCell_name ";
+ $pswarp_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF ";
+ $pswarp_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF ";
+ $pswarp_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS ";
+ $pswarp_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL ";
+ $pswarp_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID ";
+ $pswarp_command .= " -recipe PSWARP WARP -Db PSF FALSE -Db BACKGROUND.MODEL TRUE ";
+ $pswarp_command .= " -Db SOURCES FALSE ";
+ $pswarp_command .= " -log $bkgOut_log -threads 1 ";
+ $pswarp_command .= " -dbname $dbname " if defined $dbname;
+
+ # Execute
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderrr_buf ) =
+ run(command => $pswarp_command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform pswarp: $error_code", $stage, $stage_id, $error_code);
+ }
+
+ my $update_command = "$warptool ";
+ $update_command .= " -updateskyfile -warp_id $warp_id -skycell_id $skycell_id ";
+ $update_command .= " -set_background_model 1 ";
+ $update_command .= " -dbname $dbname " if defined $dbname;
+
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderrr_buf ) =
+ run(command => $update_command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform warptool: $error_code", $stage, $stage_id, $error_code);
+ }
+
+
+ }
+}
+## stack stage
+elsif ($stage eq 'stack') {
+ &my_die("--stage_id required for stage stack", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if !$stage_id;
+ my $stack_id = $stage_id; # Same as above. Alias this so I don't make mistakes.
+
+ # Configuration stuff
+ my $stacktool = can_run('stacktool') or
+ &my_die("Can't find stacktool",$stage,$stage_id,$PS_EXIT_UNKNOWN_ERROR);
+ my $ppStackMedian = can_run('ppStackMedian') or
+ &my_die("Can't find ppStackMedian",$stage,$stage_id,$PS_EXIT_UNKNOWN_ERROR);
+
+ # Get the information about this run
+ my $st_command = "$stacktool -sumskyfile -stack_id $stack_id";
+ $st_command .= " -dbname $dbname " if defined $dbname;
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $st_command, verbose => $verbose);
+ unless($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform stacktool -sumskyfile",$stage,$stage_id,$error_code);
+ }
+ if (@$stdout_buf == 0) {
+ # Nothing to do;
+ }
+ my $in_md = $mdcParser->parse(join "", @$stdout_buf) or
+ &my_die("Unable to parse metadata config doc", $stage, $stage_id, $PS_EXIT_PROG_ERROR);
+ my $stacks = parse_md_list($in_md) or
+ &my_die("Unable to parse metadata", $stage, $stage_id, $PS_EXIT_PROG_ERROR);
+ my $stack = shift(@$stacks);
+ my $path_base = $stack->{path_base};
+
+ # Get inputs
+ my $imfiles; # Array of component files
+ my $command = "$stacktool -inputskyfile -stack_id $stack_id";
+ $command .= " -dbname $dbname " if defined $dbname;
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform stacktool -inputskyfile",$stage,$stage_id,$error_code);
+ }
+ if (@$stdout_buf == 0) {
+ # Nothing to do;
+ }
+ $in_md = $mdcParser->parse(join "", @$stdout_buf) or
+ &my_die("Unable to parse metadata config doc", $stage, $stage_id, $PS_EXIT_PROG_ERROR);
+ $imfiles = parse_md_list($in_md) or
+ &my_die("Unable to parse metadata", $stage, $stage_id, $PS_EXIT_PROG_ERROR);
+
+ my $num = 0;
+ my $expTime;
+ my ($inputMDC_file, $inputMDC_name) = tempfile("/tmp/bkgreg.stack.mdc.${stage_id}.XXXX",
+ UNLINK => !$save_temps);
+ foreach my $imfile (@$imfiles) {
+ if ($imfile->{ignored}) {next; }
+ unless ($imfile->{background_model}) {
+ &my_die("Not all inputs have valid background models",
+ $stage,$stage_id,$PS_EXIT_PROG_ERROR);
+ }
+ print $inputMDC_file "INPUT${num}\tMETADATA\n";
+
+ print $inputMDC_file "\tIMAGE\tSTR\t" . $ipprc->filename( "PSWARP.OUTPUT.BKGMODEL",
+ $imfile->{path_base} ) . "\n";
+ print $inputMDC_file "\tSOURCES\tSTR\t" . $ipprc->filename( "PSWARP.OUTPUT.SOURCES",
+ $imfile->{path_base} ) . "\n";
+ print $inputMDC_file "END\n\n";
+ $num++;
+
+ $expTime = $imfile->{exp_time}; # I need something here.
+ }
+ close($inputMDC_file);
+
+ my $bkgOut_log = prepare_output("LOG.EXP",$path_base . ".BKG_REG",1);
+
+ my $ppStack_command = "$ppStackMedian -input $inputMDC_name ${path_base}.mdl ";
+ $ppStack_command .= " -recipe PPSTACK STACK -recipe PPSUB STACK -recipe PSPHOT STACK ";
+ $ppStack_command .= " -recipe PPSTATS STACKSTATS -stack-type DEEP_STACK ";
+# $ppStack_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE ";
+# $ppStack_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF ";
+# $ppStack_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS ";
+# $ppStack_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL ";
+# $ppStack_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID ";
+ $ppStack_command .= " -Db PHOTOMETRY F -Db VARIANCE F -Db CONVOLVE F ";
+ $ppStack_command .= " -Df DEFAULT.EXPTIME $expTime ";
+ $ppStack_command .= " -threads 1 -log $bkgOut_log ";
+ $ppStack_command .= " -dbname $dbname " if defined $dbname;
+
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $ppStack_command, verbose => $verbose);
+ unless($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform ppStackMedian",$stage,$stage_id,$error_code);
+ }
+
+ my $update_command = "$stacktool ";
+ $update_command .= " -updatesumskyfile -stack_id $stack_id ";
+ $update_command .= " -set_background_model 1 -fault 0 ";
+ $update_command .= " -dbname $dbname " if defined $dbname;
+
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $update_command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform stacktool: $error_code", $stage, $stage_id, $error_code);
+ }
+
+
+
+}
+## do both warp and stack
+elsif ($stage eq 'warpstack') {
+
+}
+
+sub prepare_output
+{
+ my $filerule = shift;
+ my $outroot = shift;
+ my $skycell_id = shift;
+ my $delete = shift;
+ $delete = 0 if !defined $delete;
+
+ my $error;
+ my $output = $ipprc->prepare_output($filerule, $outroot, $skycell_id, $delete, \$error)
+ or &my_die("failed to prepare output file for: $filerule", $skycell_id, $stage_id, $error);
+ return $output;
+}
+
+
+sub my_die {
+ my $msg = shift; # Warning message on die
+ my $stage = shift; # stage name
+ my $stage_id = shift; # identifier
+ my $exit_code = shift; # exit code
+
+ carp($msg);
+ exit $exit_code;
+}
+
+
+
+
+
+
+
+
+
Index: /branches/eam_branches/ipp-20121130/ippScripts/scripts/skycalibration.pl
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/scripts/skycalibration.pl (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/scripts/skycalibration.pl (revision 34839)
@@ -134,60 +134,63 @@
# First check the expected file name where the stack_id is the FILE_ID
my $file;
-if (!$singlefilter) {
+# if (!$singlefilter) {
+if (1) {
+
$file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $stack_id);
if (! $ipprc->file_resolve($file)) {
- # XXX: Beginning of section that can be removed eventually
-
- # no file with the expected name found
- # assume that the input is from an early staticsky run that did not use stack_id as the FILE_ID
- # but instead used FILE_ID = [0 .. num_filters-1]
print "\nfailed to resolve $file\n";
- if (!$filter) {
- &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
- }
- print "Trying old style FILE_ID\n";
- my $max_filters = 5;
- for (my $i=0; $i < $max_filters; $i++) {
- my $file_id = sprintf "%03d", $i;
- $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id);
- my $resolved = $ipprc->file_resolve($file);
- if (!$resolved) {
- # we fail here assumming that if file_id N doesn't exist, neither to N+1
- print "\nfailed to resolve $file\n\n";
- &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
+ # file with proper file rule not found. Try older systems
+ if ($singlefilter) {
+ # input is from a single filter static sky run which used psphot instead of psphotStack
+ # The file rule is different for psphot
+ $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base);
+ &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR)
+ unless $ipprc->file_exists($file);
+ } else {
+ # XXX: Beginning of section that can be removed eventually
+ # no file with the expected name found
+ # assume that the input is from an early multifilter staticsky run that did not use stack_id as the FILE_ID
+ # but instead used FILE_ID = [0 .. num_filters-1]
+ if (!$filter) {
+ &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
}
- # Check the filter id for this file
- my $command = "$fhead $resolved | grep FILTERID";
- my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => 0);
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR);
+ print "Trying old style FILE_ID\n";
+ my $max_filters = 5;
+ for (my $i=0; $i < $max_filters; $i++) {
+ my $file_id = sprintf "%03d", $i;
+ $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id);
+ my $resolved = $ipprc->file_resolve($file);
+ if (!$resolved) {
+ # we fail here assumming that if file_id N doesn't exist, neither to N+1
+ print "\nfailed to resolve $file\n\n";
+ &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
+ }
+ # Check the filter id for this file
+ my $command = "$fhead $resolved | grep FILTERID";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => 0);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR);
+ }
+ # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
+ my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf);
+ my $this_filter = substr $filt, 1, 7;
+
+ # if it matches we're done
+ last if $this_filter eq $filter;
+
+ # nope loop around to try the next one
+ print "Input file for $filter is not $file ($this_filter)\n";
+ $file = undef;
}
- # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
- my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf);
- my $this_filter = substr $filt, 1, 7;
-
- # if it matches we're done
- last if $this_filter eq $filter;
-
- # nope loop around to try the next one
- print "Input file for $filter is not $file ($this_filter)\n";
- $file = undef;
- }
+ }
+ # XXX: End of section that can be removed eventually
+ }
+ if ($file) {
print "\nInput file for $stack_id filter: $filter: $file\n";
-
- # XXX: End of section that can be removed eventually
- }
-} else {
- # input is from a single filter static sky run use a different file rule
- # XXX: can't we just make staticsky.pl use the same file rule?
- $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base);
- &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR)
- unless $ipprc->file_exists($file);
-}
-
-if (!$file) {
- &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR);
+ } else {
+ &my_die("Unable to find input file for stack_id $stack_id.", $skycal_id, $PS_EXIT_SYS_ERROR);
+ }
}
@@ -197,9 +200,11 @@
}
-# read the input header to find the number of detections and the number of detections
-# with extended model
+# XXX: The following is for compatability with psphotStack cmfs created prior to adding NDET
+# read the input header to find the number of detections
+my $compatability_flags = "";
my $n_detections = 0;
-my $n_extended = 0;
{
+ # NSTARS is buggy. It only counts sources with a model, and counts them twice.
+ # We want the actual number of sources (real + matched). Use the length of the psf table
my $command = "echo $resolved | $fields -n SkyChip.psf NAXIS2";
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -211,11 +216,6 @@
unless defined $n_detections;
- $command = "echo $resolved | $fields -n SkyChip.hdr NDET_EXT";
- ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
- run(command => $command, verbose => 0);
- (undef, $n_extended) = split " ", join "", @$stdout_buf;
- chomp $n_extended;
- &my_die("Unable to find number of extended objects from $file: ", $skycal_id, $PS_EXIT_SYS_ERROR)
- unless defined $n_extended;
+ $compatability_flags .= " -n_detections $n_detections";
+
}
@@ -257,4 +257,7 @@
}
+ # if ppStats didn't get n_detections use the compatability version extracted above
+ $cmdflags .= $compatability_flags unless ($cmdflags =~ /-n_detections/);
+
my $quality;
($quality) = $cmdflags =~ /-quality (\d+)/;
@@ -276,6 +279,4 @@
{
my $command = "$staticskytool -skycal_id $skycal_id";
- $command .= " -n_detections $n_detections";
- $command .= " -n_extended $n_extended";
$command .= " -addskycalresult -path_base $outroot";
$command .= " $cmdflags";
Index: /branches/eam_branches/ipp-20121130/ippScripts/scripts/skycell_jpeg.pl
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/scripts/skycell_jpeg.pl (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/scripts/skycell_jpeg.pl (revision 34839)
@@ -268,4 +268,12 @@
my %tangents = ();
+
+ my %products = ('image' => "PPSTACK.UNCONV",
+ 'mask' => "PPSTACK.UNCONV.MASK",
+ 'variance' => "PPSTACK.UNCONV.VARIANCE",
+ 'exp' => "PPSTACK.UNCONV.EXP",
+ 'num' => "PPSTACK.UNCONV.EXPNUM",
+ 'bkg' => "PPSTACK.OUTPUT.BKGMODEL"
+ );
foreach my $imfile (@$imfiles) {
@@ -281,42 +289,51 @@
$projection_cell =~ s/^(.*)\..*$/$1/;
-
+
unless (exists($tangents{$projection_cell})) {
# Make a temp file and fill, but be sure to save
- ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX",
- UNLINK => !$save_temps);
- $tangents{$projection_cell}{FILE} = $tempFile;
- $tangents{$projection_cell}{NAME} = $tempName;
- if ($masks) {
- my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX",
- UNLINK => !$save_temps);
- $tangents{$projection_cell}{MFILE} = $maskFile;
- $tangents{$projection_cell}{MNAME} = $maskName;
- }
- }
- print "$skycell_id $projection_cell\n";
- my $file = $ipprc->filename("PPSTACK.OUTPUT", $path_base, $skycell_id);
- print "$file $state $quality\n";
- my $f_fh = $tangents{$projection_cell}{FILE};
- print $f_fh "$file\n";
- if ($masks) {
- my $mask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base, $skycell_id);
- print "$mask\n";
- my $m_fh = $tangents{$projection_cell}{MFILE};
- print $m_fh "$mask\n";
+ foreach my $key (keys %products) {
+ ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.$key.XXXX",
+ UNLINK => !$save_temps);
+ $tangents{$projection_cell}{$key}{FILE} = $tempFile;
+ $tangents{$projection_cell}{$key}{NAME} = $tempName;
+ }
+ }
+ foreach my $key (keys %products) {
+ print "$skycell_id $projection_cell\n";
+ my $file = $ipprc->filename($products{$key}, $path_base, $skycell_id);
+ print "$file $state $quality\n";
+ my $f_fh = $tangents{$projection_cell}{$key}{FILE};
+ print $f_fh "$file\n";
}
}
foreach my $projection_cell (keys %tangents) {
- $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}";
- if ($masks) {
- $command .= " -masks $tangents{$projection_cell}{MNAME} ";
- }
- $command .= " ${outroot}.${projection_cell} ";
- print "$command\n";
- ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
- unless ($success) {
- $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
- &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code);
- }
+ ## Loop over results here.
+ # Images
+ # Masks
+ # Variances
+ # Nexptime
+ # Nexp
+ # Backgrounds
+
+ foreach my $key (keys %products) {
+ $command = "$ppSkycell -images $tangents{$projection_cell}{$key}{NAME}";
+ $command .= " ${outroot}.${projection_cell}.${key} ";
+ if ($key eq 'bkg') {
+ $command .= " -Di BIN1 1 -Di BIN2 1 ";
+ }
+ elsif ($key eq 'image') {
+ $command .= " -masks $tangents{$projection_cell}{mask}{NAME} ";
+ }
+ elsif ($key eq 'mask') {
+ next; # This should be made with the images.
+ }
+ print "$command\n";
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+ &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code);
+ }
+ }
+
# Update database:
$command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot";
Index: /branches/eam_branches/ipp-20121130/ippScripts/scripts/stack_skycell.pl
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/scripts/stack_skycell.pl (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/scripts/stack_skycell.pl (revision 34839)
@@ -229,4 +229,6 @@
my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name
+ my $bkgmodel = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL", $file->{path_base});
+
&my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image );
&my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask );
@@ -240,4 +242,5 @@
print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve;
print $listFile "\tSOURCES\tSTR\t" . $sources . "\n";
+ print $listFile "\tBKGMODEL\tSTR\t" . $bkgmodel . "\n" if $ipprc->file_exists( $bkgmodel );
print $listFile "END\n\n";
Index: /branches/eam_branches/ipp-20121130/ippScripts/scripts/warp_skycell.pl
===================================================================
--- /branches/eam_branches/ipp-20121130/ippScripts/scripts/warp_skycell.pl (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippScripts/scripts/warp_skycell.pl (revision 34839)
@@ -128,4 +128,5 @@
# Where do we get the astrometry source from?
my $astromSource; # The astrometry source
+my $doBackground; # Do we want to make background models?
{
my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -";
@@ -139,4 +140,5 @@
&my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
$astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
+ $doBackground = metadataLookupBool($metadata, 'BACKGROUND.MODEL');
}
@@ -168,4 +170,8 @@
if ($do_stats) {
$outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1) if $do_stats;
+}
+my $outputBKGs;
+if ($doBackground) {
+ $outputBKGs = prepare_output ("PSWARP.OUTPUT.BKGMODEL", $outroot, $skycell_id, 1);
}
my $configuration;
@@ -209,5 +215,8 @@
my ($weightFile, $weightName) = tempfile( "$tempOutRoot.weight.list.XXXX", UNLINK => !$save_temps);
my ($astromFile, $astromName) = tempfile( "$tempOutRoot.astrom.list.XXXX", UNLINK => !$save_temps);
-
+my ($bkgFile, $bkgName);
+if ($doBackground) {
+ ($bkgFile, $bkgName) = tempfile( "$tempOutRoot.bkg.list.XXXX", UNLINK => !$save_temps);
+}
my $wrote_astrom = 0;
foreach my $imfile (@$imfiles) {
@@ -235,4 +244,9 @@
print $maskFile "$mask\n";
print $weightFile "$weight\n";
+ my $bkg;
+ if ($doBackground) {
+ $bkg = $ipprc->filename("PSPHOT.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id});
+ print $bkgFile "$bkg\n";
+ }
if (!$wrote_astrom) {
@@ -245,5 +259,7 @@
close $weightFile;
close $astromFile;
-
+if ($doBackground) {
+ close($bkgFile);
+}
# We need the recipe to determine if we care whether the PSF is generated or not
my $recipe;
@@ -269,4 +285,5 @@
$command .= " -variancelist $weightName";
$command .= " -astromlist $astromName";
+ $command .= " -bkglist $bkgName" if ($doBackground);
$command .= " $outroot $skyFile";
$command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE";
Index: /branches/eam_branches/ipp-20121130/ippTasks/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/Makefile.am (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTasks/Makefile.am (revision 34839)
@@ -45,5 +45,6 @@
diffphot.pro \
lap.pro \
- vp.pro
+ vp.pro \
+ bg.regeneration.pro
other_files = \
Index: /branches/eam_branches/ipp-20121130/ippTasks/addstar.pro
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/addstar.pro (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTasks/addstar.pro (revision 34839)
@@ -212,5 +212,5 @@
host local
periods -poll $LOADPOLL
- periods -exec 300
+ periods -exec $LOADEXEC_ADD
periods -timeout 300
npending 1
@@ -266,5 +266,5 @@
host local
periods -poll $LOADPOLL
- periods -exec 300
+ periods -exec $LOADEXEC_ADD
periods -timeout 300
npending 1
Index: /branches/eam_branches/ipp-20121130/ippTasks/bg.regeneration.pro
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/bg.regeneration.pro (revision 34839)
+++ /branches/eam_branches/ipp-20121130/ippTasks/bg.regeneration.pro (revision 34839)
@@ -0,0 +1,306 @@
+## bg.regeneration.pro : tasks for regenerating the background model for warps and stacks : -*- sh -*-
+
+check.globals
+
+if ($?POLL_LIMIT_BGREG == 0) set POLL_LIMIT_BGREG = 40
+
+macro set.bgreg.poll
+ if ($0 != 2)
+ echo "USAGE:set.bgreg.poll (value)"
+ break;
+ end
+
+ $POLL_LIMIT_BGREG = $1
+end
+macro get.bgreg.poll
+ echo $POLL_LIMIT_BGREG
+end
+
+## Initialize the books for the tasks to do
+book init bgRegWarp
+book init bgRegStack
+
+## Database lists
+$bgRegWarp_DB = 0
+$bgRegStack_DB = 0
+
+macro bgreg.warp.status
+ book listbook bgRegWarp
+end
+macro bgreg.stack.status
+ book listbook bgRegStack
+end
+
+## Reset tasks
+macro bgreg.reset
+ book init bgRegWarp
+ book init bgRegStack
+end
+
+## Turn tasks on
+macro bgreg.on
+ task bgreg.warp.load
+ active true
+ end
+ task bgreg.warp.run
+ active true
+ end
+ task bgreg.stack.load
+ active true
+ end
+ task bgreg.stack.run
+ active true
+ end
+end
+## or off
+macro bgreg.off
+ task bgreg.warp.load
+ active false
+ end
+ task bgreg.warp.run
+ active false
+ end
+ task bgreg.stack.load
+ active false
+ end
+ task bgreg.stack.run
+ active false
+ end
+end
+
+# Load task for warp
+task bgreg.warp.load
+ host local
+
+ periods -poll $LOADPOLL
+ periods -poll $LOADEXEC
+ periods -timeout 30
+ npending 1
+
+ stdout NULL
+ stderr $LOGDIR/bgreg.warp.load
+
+ task.exec
+ if ($LABEL:n == 0) break
+ $run = warptool -warped -background_model 0
+ if ($DB:n == 0)
+ option DEFAULT
+ else
+ #save the DB naem for the exit tasks
+ option $DB:$bgRegWarp_DB
+ $run = $run -dbname $DB:$bgRegWarp_DB
+ $bgRegWarp_DB ++
+ if ($bgRegWarp_DB >= $DB:n) set bgRegWarp_DB = 0
+ end
+ add_poll_args run
+ add_poll_labels run
+ # change the limit?
+ $run = $run -limit $POLL_LIMIT_BGREG
+ command $run
+ end
+
+ # success
+ task.exit 0
+ # convert 'stdout' to book format
+ ipptool2book stdout bgRegWarp -key warp_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+ if ($VERBOSE > 2)
+ book listbook bgRegWarp
+ end
+
+ # delete existing entries in the appropriate pantaskStates
+ process_cleanup bgRegWarp
+ end
+
+ # locked list
+ task.exit default
+ showcommand failure
+ end
+
+ task.exit crash
+ showcommand crash
+ end
+
+ # operation times out?
+ task.exit timeout
+ showcommand timeout
+ end
+end
+
+task bgreg.warp.run
+ periods -poll $RUNPOLL
+ periods -exec $RUNEXEC
+ periods -timeout 60
+
+ task.exec
+ # if we can't exec, set the retry time long
+ periods -exec $RUNEXEC
+
+ book npages bgRegWarp -var N
+ if ($N == 0) break
+ if ($NETWORK == 0) break
+
+ # look for new pages
+ book getpage bgRegWarp 0 -var pageName -key pantaskState INIT
+ if ("$pageName" == "NULL") break
+
+ book setword bgRegWarp $pageName pantaskState RUN
+ book getword bgRegWarp $pageName warp_id -var STAGE_ID
+ book getword bgRegWarp $pageName skycell_id -var SKYCELL_ID
+ book getword bgRegWarp $pageName camera -var CAMERA
+ book getword bgRegWarp $pageName dbname -var DBNAME
+
+ set.host.for.skycell $SKYCELL_ID
+
+ stdout $LOGDIR/bgreg.warp.log
+ stderr $LOGDIR/bgreg.warp.log
+
+ $run = regenerate_background.pl --stage warp --stage_id $STAGE_ID --skycell_id $SKYCELL_ID --camera $CAMERA
+
+ add_standard_args run
+ options $pageName
+
+ if ($VERBOSE > 1)
+ echo command $run
+ end
+ periods -exec 0.05
+ command $run
+ end
+
+ # default exit status
+ task.exit default
+ process_exit bgRegWarp $options:0 $JOB_STATUS
+ end
+
+ # locked list
+ task.exit crash
+ showcommand crash
+ echo "hostname: $JOB_HOSTNAME"
+ book setword bgRegWarp $options:0 pantasksState CRASH
+ end
+
+ # operation timed out
+ task.exit timeout
+ showcommand timeout
+ book setword bgRegWarp $options:0 pantaskState TIMEOUT
+ end
+end
+
+# Load task for stack
+task bgreg.stack.load
+ host local
+
+ periods -poll $LOADPOLL
+ periods -poll $LOADEXEC
+ periods -timeout 30
+ npending 1
+
+ stdout NULL
+ stderr $LOGDIR/bgreg.stack.load
+
+ task.exec
+ if ($LABEL:n == 0) break
+ $run = stacktool -tobkg
+ if ($DB:n == 0)
+ option DEFAULT
+ else
+ #save the DB naem for the exit tasks
+ option $DB:$bgRegStack_DB
+ $run = $run -dbname $DB:$bgRegStack_DB
+ $bgRegStack_DB ++
+ if ($bgRegStack_DB >= $DB:n) set bgRegStack_DB = 0
+ end
+ add_poll_args run
+ add_poll_labels run
+ # change the limit?
+ $run = $run -limit $POLL_LIMIT_BGREG
+ command $run
+ end
+
+ # success
+ task.exit 0
+ # convert 'stdout' to book format
+ ipptool2book stdout bgRegStack -key stack_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+ if ($VERBOSE > 2)
+ book listbook bgRegStack
+ end
+
+ # delete existing entries in the appropriate pantaskStates
+ process_cleanup bgRegStack
+ end
+
+ # locked list
+ task.exit default
+ showcommand failure
+ end
+
+ task.exit crash
+ showcommand crash
+ end
+
+ # operation times out?
+ task.exit timeout
+ showcommand timeout
+ end
+end
+
+task bgreg.stack.run
+ periods -poll $RUNPOLL
+ periods -exec $RUNEXEC
+ periods -timeout 60
+
+ task.exec
+ # if we can't exec, set the retry time long
+ periods -exec $RUNEXEC
+
+ book npages bgRegStack -var N
+ if ($N == 0) break
+ if ($NETWORK == 0) break
+
+ # look for new pages
+ book getpage bgRegStack 0 -var pageName -key pantaskState INIT
+ if ("$pageName" == "NULL") break
+
+ book setword bgRegStack $pageName pantaskState RUN
+ book getword bgRegStack $pageName stack_id -var STAGE_ID
+ book getword bgRegStack $pageName skycell_id -var SKYCELL_ID
+ book getword bgRegStack $pageName camera -var CAMERA
+ book getword bgRegStack $pageName dbname -var DBNAME
+ set.host.for.skycell $SKYCELL_ID
+
+ stdout $LOGDIR/bgreg.stack.log
+ stderr $LOGDIR/bgreg.stack.log
+
+ $run = regenerate_background.pl --stage stack --stage_id $STAGE_ID --skycell_id $SKYCELL_ID --camera $CAMERA
+
+ add_standard_args run
+ options $pageName
+
+ if ($VERBOSE > 1)
+ echo command $run
+ end
+ periods -exec 0.05
+ command $run
+ end
+
+ # default exit status
+ task.exit default
+ process_exit bgRegStack $options:0 $JOB_STATUS
+ end
+
+ # locked list
+ task.exit crash
+ showcommand crash
+ echo "hostname: $JOB_HOSTNAME"
+ book setword bgRegStack $options:0 pantasksState CRASH
+ end
+
+ # operation timed out
+ task.exit timeout
+ showcommand timeout
+ book setword bgRegStack $options:0 pantaskState TIMEOUT
+ end
+end
+
+
+
+
Index: /branches/eam_branches/ipp-20121130/ippTasks/nightly_science.pro
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/nightly_science.pro (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTasks/nightly_science.pro (revision 34839)
@@ -258,5 +258,5 @@
task ns.dqstats.load
host local
- periods -poll 3600
+ periods -poll $LOADPOLL
periods -exec $LOADEXEC
periods -timeout 300
Index: /branches/eam_branches/ipp-20121130/ippTasks/science.cleanup.pro
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/science.cleanup.pro (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTasks/science.cleanup.pro (revision 34839)
@@ -13,4 +13,17 @@
check.globals
+
+# if set tell cleanup script to check for files from all components regardless of the data_state
+$CHECK_ALL_COMPONENTS = 0
+macro set.check.all.components
+ $CHECK_ALL_COMPONENTS = 1
+end
+macro clear.check.all.components
+ $CHECK_ALL_COMPONENTS = 0
+end
+macro get.check.all.components
+ echo CHECK_ALL_COMPONENTS = $CHECK_ALL_COMPONENTS
+end
+
## chip.cleanup.pro : -*- sh -*-
@@ -130,14 +143,22 @@
book getword chipPendingCleanup $pageName state -var CLEANUP_MODE
book getword chipPendingCleanup $pageName chip_id -var CHIP_ID
+ book getword chipPendingCleanup $pageName exp_tag -var EXP_TAG
+ book getword chipPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
book getword chipPendingCleanup $pageName dbname -var DBNAME
- # specify choice of local or remote host based on camera and chip (class_id)
+ # specify choice of local or remote host based on camera and chip (in this case FPA)
set.host.for.camera $CAMERA FPA
+
+ set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
+
+ sprintf LOGFILE "%s/%s/%s.ch.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID
stdout $LOGDIR/chip.cleanup.log
stderr $LOGDIR/chip.cleanup.log
- # XXX is everything listed here needed?
- $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE
+ $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+ if ($CHECK_ALL_COMPONENTS)
+ $run = $run --check-all
+ end
add_standard_args run
@@ -282,4 +303,6 @@
book getword camPendingCleanup $pageName state -var CLEANUP_MODE
book getword camPendingCleanup $pageName cam_id -var CAM_ID
+ book getword camPendingCleanup $pageName exp_tag -var EXP_TAG
+ book getword camPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
book getword camPendingCleanup $pageName dbname -var DBNAME
@@ -287,9 +310,14 @@
set.host.for.camera $CAMERA FPA
+ # set the WORKDIR variable
+ set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
+
+ ## generate LOGFILE specific to this exposure (& cam_id)
+ sprintf LOGFILE "%s/%s/%s.cm.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID
+
stdout $LOGDIR/camera.cleanup.log
stderr $LOGDIR/camera.cleanup.log
- # XXX is everything listed here needed?
- $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE
+ $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
add_standard_args run
@@ -591,14 +619,21 @@
book getword warpPendingCleanup $pageName state -var CLEANUP_MODE
book getword warpPendingCleanup $pageName warp_id -var WARP_ID
+ book getword warpPendingCleanup $pageName exp_tag -var EXP_TAG
+ book getword warpPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
book getword warpPendingCleanup $pageName dbname -var DBNAME
# specify choice of local or remote host based on camera and warp (class_id)
set.host.for.camera $CAMERA FPA
+ set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+ sprintf LOGFILE "%s/%s/%s.wrp.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID
stdout $LOGDIR/warp.cleanup.log
stderr $LOGDIR/warp.cleanup.log
- # XXX is everything listed here needed?
- $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE
+ $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+ if ($CHECK_ALL_COMPONENTS)
+ $run = $run --check-all
+ end
add_standard_args run
@@ -748,14 +783,21 @@
book getword diffCleanup $pageName state -var CLEANUP_MODE
book getword diffCleanup $pageName diff_id -var DIFF_ID
+ book getword diffCleanup $pageName tess_id -var TESS_ID
+ book getword diffCleanup $pageName workdir -var WORKDIR_TEMPLATE
book getword diffCleanup $pageName dbname -var DBNAME
# specify choice of local or remote host based on camera and diff (class_id)
set.host.for.camera $CAMERA FPA
+ set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+ sprintf LOGFILE "%s/%s/%s.dif.%s.cleanup.log" $WORKDIR $TESS_ID $TESS_ID $DIFF_ID
+
stdout $LOGDIR/diff.cleanup.log
stderr $LOGDIR/diff.cleanup.log
- # XXX is everything listed here needed?
- $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE
+ $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+ if ($CHECK_ALL_COMPONENTS)
+ $run = $run --check-all
+ end
add_standard_args run
@@ -899,4 +941,5 @@
book getword stackCleanup $pageName state -var CLEANUP_MODE
book getword stackCleanup $pageName stack_id -var STACK_ID
+ book getword stackCleanup $pageName path_base -var PATH_BASE
book getword stackCleanup $pageName dbname -var DBNAME
@@ -907,6 +950,6 @@
stderr $LOGDIR/stack.cleanup.log
- # XXX is everything listed here needed?
- $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE
+ sprintf LOGFILE "%s.cleanup.log" PATH_BASE
+ $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
add_standard_args run
Index: /branches/eam_branches/ipp-20121130/ippTasks/skycal.pro
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/skycal.pro (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTasks/skycal.pro (revision 34839)
@@ -8,4 +8,20 @@
# test for required global variables
check.globals
+
+# skcal.pro should have a more restricted polling limit (to avoid stress with getstar)
+if ($?POLL_LIMIT_SKYCAL == 0) set POLL_LIMIT_SKYCAL = 25
+
+macro set.skycal.poll
+ if ($0 != 2)
+ echo "USAGE:set.skycal.poll (value)"
+ break
+ end
+
+ $POLL_LIMIT_SKYCAL = $1
+end
+
+macro get.skycal.poll
+ echo "skycal poll limit: $POLL_LIMIT_SKYCAL"
+end
### Initialise the books containing the tasks to do
@@ -79,5 +95,5 @@
# save the DB name for the exit tasks
option $DB:$skycal_DB
- $run = $run -dbname $DB:$skycal_DB -limit 40
+ $run = $run -dbname $DB:$skycal_DB
$skycal_DB ++
if ($skycal_DB >= $DB:n) set skycal_DB = 0
@@ -85,4 +101,5 @@
add_poll_args run
add_poll_labels run
+ $run = $run -limit $POLL_LIMIT_SKYCAL
command $run
end
@@ -123,8 +140,11 @@
task.exec
+ # if we are unable to run use "long" exectime
+ periods -exec $RUNEXEC
book npages skycalRun -var N
if ($N == 0) break
if ($NETWORK == 0) break
if ($BURNTOOLING == 1) break
+
# look for new entries in skycalRun (pantaskState == INIT)
@@ -147,5 +167,6 @@
# set the host and workdir based on the skycell hash
- set.host.for.skycell $SKYCELL_ID
+ # set.host.for.skycell $SKYCELL_ID
+ host anyhost
set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
@@ -172,4 +193,6 @@
echo command $run
end
+ # since we have work to do shorten exec time
+ periods -exec .1
command $run
end
@@ -197,9 +220,9 @@
host local
- periods -poll 60.0
- periods -exec 1800.0
+ periods -poll 10.0
+ periods -exec 1200.0
periods -timeout 120.0
npending 1
- active false
+ active true
stdout NULL
Index: /branches/eam_branches/ipp-20121130/ippTasks/staticsky.pro
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/staticsky.pro (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTasks/staticsky.pro (revision 34839)
@@ -70,6 +70,6 @@
periods -poll $LOADPOLL
- periods -exec $LOADEXEC
- periods -timeout 30
+ periods -exec 30
+ periods -timeout 60
npending 1
@@ -154,5 +154,6 @@
# set the host and workdir based on the skycell hash
- set.host.for.skycell $SKYCELL_ID
+ # set.host.for.skycell $SKYCELL_ID
+ host anyhost
set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
Index: /branches/eam_branches/ipp-20121130/ippTasks/summit.copy.pro
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/summit.copy.pro (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTasks/summit.copy.pro (revision 34839)
@@ -223,5 +223,5 @@
periods -exec 10
periods -poll 1
- periods -timeout 20
+ periods -timeout 60
npending 1
Index: /branches/eam_branches/ipp-20121130/ippTasks/survey.pro
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTasks/survey.pro (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTasks/survey.pro (revision 34839)
@@ -17,4 +17,5 @@
book create SURVEY_PUBLISH
book create SURVEY_STATICSKYSINGLE
+ book create SURVEY_SKYCAL
$haveSurveyBooks = TRUE
end
@@ -31,4 +32,5 @@
$SURVEY_WARP_BG_DB = 0
$SURVEY_PUBLISH_DB = 0
+$SURVEY_SKYCAL_DB = 0
$SURVEY_STATICSKYSINGLE_DB = 0
$SURVEY_EXEC = 120
@@ -73,4 +75,7 @@
active true
end
+ task survey.skycal
+ active true
+ end
end
@@ -110,4 +115,7 @@
end
task survey.staticskysingle
+ active false
+ end
+ task survey.skycal
active false
end
@@ -464,4 +472,32 @@
book delpage SURVEY_STATICSKYSINGLE $1
end
+
+macro survey.add.skycal
+ if ($0 != 3)
+ echo "USAGE: survey.add.skycal (label) (dist_group)"
+ break
+ end
+ book newpage SURVEY_SKYCAL $1
+ book setword SURVEY_SKYCAL $1 LABEL $1
+ book setword SURVEY_SKYCAL $1 DIST_GROUP $2
+ book setword SURVEY_SKYCAL $1 STATE PENDING
+end
+
+macro survey.del.skycal
+ if ($0 != 2)
+ echo "USAGE: survey.del.skycal (label)"
+ break
+ end
+ book delpage SURVEY_SKYCAL $1
+end
+
+macro survey.show.skycal
+ if ($0 != 1)
+ echo "USAGE: survey.show.skyacl"
+ break
+ end
+ book listbook SURVEY_SKYCAL
+end
+
task survey.diff
@@ -1453,2 +1489,74 @@
end
+# Survey task for Skycal stage
+
+task survey.skycal
+ host local
+
+ periods -poll $SURVEY_POLL
+ periods -exec $SURVEY_EXEC
+ periods -timeout $SURVEY_TIMEOUT
+ npending 1
+
+ stdout $LOGDIR/survey.skycal.log
+ stderr $LOGDIR/survey.skycal.log
+
+ task.exec
+ book npages SURVEY_SKYCAL -var N
+ if ($N == 0)
+# echo "No labels for processing"
+ break
+ endif
+
+ book getpage SURVEY_SKYCAL 0 -var label -key STATE NEW
+ if ("$label" == "NULL")
+ # All labels have been done --- reset
+# echo "Resetting labels"
+ for i 0 $N
+ book getpage SURVEY_SKYCAL $i -var label
+ book setword SURVEY_SKYCAL $label STATE NEW
+ end
+ book getpage SURVEY_SKYCAL 0 -var label -key STATE NEW
+
+ # Select different database
+ $SURVEY_SKYCAL_DB ++
+ if ($SURVEY_SKYCAL_DB >= $DB:n) set SURVEY_SKYCAL_DB = 0
+ end
+
+ book setword SURVEY_SKYCAL $label STATE DONE
+ book getword SURVEY_SKYCAL $label DIST_GROUP -var dist_group
+
+ # note: currently skycal uses the staticskyRun workdir
+
+ $run = staticskytool -defineskycalrun -select_label $label -set_dist_group $dist_group
+
+ if ($DB:n == 0)
+ option DEFAULT
+ else
+ $run = $run -dbname $DB:$SURVEY_SKYCAL_DB
+ option $DB:$SURVEY_SKYCAL_DB
+ end
+
+ # echo $run
+ command $run
+ end
+
+ # success
+ task.exit 0
+# echo "Success"
+ end
+
+ # locked list
+ task.exit default
+ showcommand failure
+ end
+
+ task.exit crash
+ showcommand crash
+ end
+
+ # operation times out?
+ task.exit timeout
+ showcommand timeout
+ end
+end
Index: /branches/eam_branches/ipp-20121130/ippToPsps/config/OB/tables.vot
===================================================================
--- /branches/eam_branches/ipp-20121130/ippToPsps/config/OB/tables.vot (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippToPsps/config/OB/tables.vot (revision 34839)
@@ -27,6 +27,6 @@
flag indicating odd properties
-
- variable in difference image detections
+
+ subset of objInfoFlag - flags related to goodness of object
@@ -45,4 +45,13 @@
Declination error
+
+ Cartesian x
+
+
+ Cartesian y
+
+
+ Cartesian z
+
ecliptic longitude
@@ -284,4 +293,7 @@
Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation
+
+
+ Random number between 0 and 1
Index: /branches/eam_branches/ipp-20121130/ippToPsps/jython/objectbatch.py
===================================================================
--- /branches/eam_branches/ipp-20121130/ippToPsps/jython/objectbatch.py (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippToPsps/jython/objectbatch.py (revision 34839)
@@ -134,5 +134,5 @@
," + filter[1] + "80pct = MAG_80/1000 \
," + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) \
- ," + filter[1] + "StackDetectID = STACK_DETECT_ID + " + str(filterID)
+ ," + filter[1] + "StackDetectID = STACK_DETECT_ID"
#self.logger.info(sql)
@@ -236,5 +236,5 @@
,ippObjID \
,objInfoFlag \
- ,varFlag \
+ ,qualityFlag \
,surveyID \
,ra \
@@ -246,4 +246,5 @@
,sgSep \
,dataRelease \
+ ,Random \
) \
SELECT \
@@ -251,5 +252,5 @@
,CAT_ID*1000000000 + OBJ_ID \
,FLAGS \
- ,0 \
+ ,FLAGS >> 24 & 0xFF \
," + str(self.surveyID) + " \
,RA \
@@ -257,8 +258,9 @@
,RA_ERR \
,DEC_ERR \
- ,NMEASURE \
+ ,0 \
,PSF_QF_PERF \
,STARGAL_SEP \
, " + str(self.config.dataRelease) + "\
+ , RAND() \
FROM " + cptTableName
Index: /branches/eam_branches/ipp-20121130/ippToPsps/jython/stackbatch.py
===================================================================
--- /branches/eam_branches/ipp-20121130/ippToPsps/jython/stackbatch.py (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippToPsps/jython/stackbatch.py (revision 34839)
@@ -5,4 +5,5 @@
import stilts
+import re
from java.lang import *
from java.sql import *
@@ -143,49 +144,134 @@
'''
def updateApFlxs(self, prefix, psfCondition):
+ #properly -999 these to start with:
sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \
- "+prefix+"flxR1=b.APER_FLUX_1 / " + str(self.expTime) + ", \
- "+prefix+"flxR1Err=b.APER_FLUX_ERR_1 / " + str(self.expTime) + ", \
- "+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 / " + str(self.expTime) + ", \
- "+prefix+"flxR1Fill=b.APER_FILL_1, \
- "+prefix+"flxR2=b.APER_FLUX_2 / " + str(self.expTime) + ", \
- "+prefix+"flxR2Err=b.APER_FLUX_ERR_2 / " + str(self.expTime) + ", \
- "+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 / " + str(self.expTime) + ", \
- "+prefix+"flxR2Fill=b.APER_FILL_2, \
- "+prefix+"flxR3=b.APER_FLUX_3 / " + str(self.expTime) + ", \
- "+prefix+"flxR3Err=b.APER_FLUX_ERR_3 / " + str(self.expTime) + ", \
- "+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 / " + str(self.expTime) + ", \
- "+prefix+"flxR3Fill=b.APER_FILL_3, \
- "+prefix+"flxR4=b.APER_FLUX_4 / " + str(self.expTime) + ", \
- "+prefix+"flxR4Err=b.APER_FLUX_ERR_4 / " + str(self.expTime) + ", \
- "+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 / " + str(self.expTime) + ", \
- "+prefix+"flxR4Fill=b.APER_FILL_4, \
- "+prefix+"flxR5=b.APER_FLUX_5 / " + str(self.expTime) + ", \
- "+prefix+"flxR5Err=b.APER_FLUX_ERR_5 / " + str(self.expTime) + ", \
- "+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 / " + str(self.expTime) + ", \
- "+prefix+"flxR5Fill=b.APER_FILL_5, \
- "+prefix+"flxR6=b.APER_FLUX_6 / " + str(self.expTime) + ", \
- "+prefix+"flxR6Err=b.APER_FLUX_ERR_6 / " + str(self.expTime) + ", \
- "+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 / " + str(self.expTime) + ", \
- "+prefix+"flxR6Fill=b.APER_FILL_6, \
- "+prefix+"flxR7=b.APER_FLUX_7 / " + str(self.expTime) + ", \
- "+prefix+"flxR7Err=b.APER_FLUX_ERR_7 / " + str(self.expTime) + ", \
- "+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 / " + str(self.expTime) + ", \
- "+prefix+"flxR7Fill=b.APER_FILL_7, \
- "+prefix+"flxR8=b.APER_FLUX_8 / " + str(self.expTime) + ", \
- "+prefix+"flxR8Err=b.APER_FLUX_ERR_8 / " + str(self.expTime) + ", \
- "+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 / " + str(self.expTime) + ", \
- "+prefix+"flxR8Fill=b.APER_FILL_8, \
- "+prefix+"flxR9=b.APER_FLUX_9 / " + str(self.expTime) + ", \
- "+prefix+"flxR9Err=b.APER_FLUX_ERR_9 / " + str(self.expTime) + ", \
- "+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 / " + str(self.expTime) + ", \
- "+prefix+"flxR9Fill=b.APER_FILL_9, \
- "+prefix+"flxR10=b.APER_FLUX_10 / " + str(self.expTime) + ", \
- "+prefix+"flxR10Err=b.APER_FLUX_ERR_10 / " + str(self.expTime) + ", \
- "+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 / " + str(self.expTime) + ", \
- "+prefix+"flxR10Fill=b.APER_FILL_10 \
- WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
-
- self.scratchDb.execute(sql)
+ "+prefix+"flxR1=-999, \
+ "+prefix+"flxR1Err=-999, \
+ "+prefix+"flxR1Std=-999, \
+ "+prefix+"flxR1Fill=-999, \
+ "+prefix+"flxR2=-999, \
+ "+prefix+"flxR2Err=-999, \
+ "+prefix+"flxR2Std=-999, \
+ "+prefix+"flxR2Fill=-999, \
+ "+prefix+"flxR3=-999, \
+ "+prefix+"flxR3Err=-999, \
+ "+prefix+"flxR3Std=-999, \
+ "+prefix+"flxR3Fill=-999, \
+ "+prefix+"flxR4=-999, \
+ "+prefix+"flxR4Err=-999, \
+ "+prefix+"flxR4Std=-999, \
+ "+prefix+"flxR4Fill=-999, \
+ "+prefix+"flxR5=-999, \
+ "+prefix+"flxR5Err=-999, \
+ "+prefix+"flxR5Std=-999, \
+ "+prefix+"flxR5Fill=-999, \
+ "+prefix+"flxR6=-999, \
+ "+prefix+"flxR6Err=-999, \
+ "+prefix+"flxR6Std=-999, \
+ "+prefix+"flxR6Fill=-999, \
+ "+prefix+"flxR7=-999, \
+ "+prefix+"flxR7Err=-999, \
+ "+prefix+"flxR7Std=-999, \
+ "+prefix+"flxR7Fill=-999, \
+ "+prefix+"flxR8=-999, \
+ "+prefix+"flxR8Err=-999, \
+ "+prefix+"flxR8Std=-999, \
+ "+prefix+"flxR8Fill=-999, \
+ "+prefix+"flxR9=-999, \
+ "+prefix+"flxR9Err=-999, \
+ "+prefix+"flxR9Std=-999, \
+ "+prefix+"flxR9Fill=-999, \
+ "+prefix+"flxR10=-999, \
+ "+prefix+"flxR10Err=-999, \
+ "+prefix+"flxR10Std=-999, \
+ "+prefix+"flxR10Fill=-999 \
+ WHERE a.ippDetectID=b.IPP_IDET "
+ #self.logger.infoPair("sql", sql)
+ self.scratchDb.execute(sql)
+
+
+ #we have variable numbers of these columns. find out which are in use
+ sql = "DESCRIBE SkyChip_xrad 'aper_fill_%'"
+ rs = self.scratchDb.executeQuery(sql)
+ rs.first()
+ columns = []
+ column = rs.getString(1)
+ columns.append(column)
+ while (rs.next()):
+ column = rs.getString(1)
+ columns.append(column)
+ rs.close()
+
+ #use regex to leave just the numbers
+
+ numbers = []
+ for column in columns:
+ number = re.sub("APER_FILL_","",column)
+ if (int(number) >= 1 and int(number) <=10):
+ numbers.append(number)
+ self.logger.infoPair("Aperture Numbers",number)
+
+ #generate the sql to do the necessary ops on the columns
+ sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET "
+ cnt =0
+ for number in numbers:
+ if (cnt>0):
+ sql=sql+" , "
+ sql=sql+prefix+"flxR"+number+"=b.APER_FLUX_"+number +" / " + str(self.expTime) + ", "
+ sql=sql+prefix+"flxR"+number+"Err=b.APER_FLUX_ERR_"+number +" / " + str(self.expTime) + ", "
+ sql=sql+prefix+"flxR"+number+"Std=b.APER_FLUX_STDEV_"+number +" / " + str(self.expTime) + ", "
+ sql=sql+prefix+"flxR"+number+"Fill=b.APER_FILL_"+number + " "
+ cnt=cnt+1
+
+ sql=sql+"WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
+ self.scratchDb.execute(sql)
+
+ #self.logger.infoPair("sql",sql)
+
+ #sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \
+ #"+prefix+"flxR1=b.APER_FLUX_1 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR1Err=b.APER_FLUX_ERR_1 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR1Fill=b.APER_FILL_1, \
+ #"+prefix+"flxR2=b.APER_FLUX_2 / " + str(self.expTime) + ", \
+ ##"+prefix+"flxR2Err=b.APER_FLUX_ERR_2 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR2Fill=b.APER_FILL_2, \
+ #"+prefix+"flxR3=b.APER_FLUX_3 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR3Err=b.APER_FLUX_ERR_3 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR3Fill=b.APER_FILL_3, \
+ #"+prefix+"flxR4=b.APER_FLUX_4 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR4Err=b.APER_FLUX_ERR_4 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR4Fill=b.APER_FILL_4, \
+ #"+prefix+"flxR5=b.APER_FLUX_5 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR5Err=b.APER_FLUX_ERR_5 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR5Fill=b.APER_FILL_5, \
+ #"+prefix+"flxR6=b.APER_FLUX_6 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR6Err=b.APER_FLUX_ERR_6 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR6Fill=b.APER_FILL_6, \
+ #"+prefix+"flxR7=b.APER_FLUX_7 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR7Err=b.APER_FLUX_ERR_7 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR7Fill=b.APER_FILL_7, \
+ #"+prefix+"flxR8=b.APER_FLUX_8 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR8Err=b.APER_FLUX_ERR_8 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR8Fill=b.APER_FILL_8, \
+ #"+prefix+"flxR9=b.APER_FLUX_9 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR9Err=b.APER_FLUX_ERR_9 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR9Fill=b.APER_FILL_9, \
+ #"+prefix+"flxR10=b.APER_FLUX_10 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR10Err=b.APER_FLUX_ERR_10 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 / " + str(self.expTime) + ", \
+ #"+prefix+"flxR10Fill=b.APER_FILL_10 \
+ #WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
+
+ #self.scratchDb.execute(sql)
'''
@@ -395,5 +481,5 @@
,KRON_FLUX_ERR / " + str(self.expTime) + " \
,MOMENTS_R1 * 2.5 \
- ,PSF_CHISQ \
+ ,PSF_CHISQ / PSF_NDOF \
,N_FRAMES \
, '" + self.dateStr + "' \
Index: /branches/eam_branches/ipp-20121130/ippToPsps/src/Dvo.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippToPsps/src/Dvo.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippToPsps/src/Dvo.c (revision 34839)
@@ -21,5 +21,7 @@
if (this->mysql) mysql_close(this->mysql);
- free(this);
+ // libohana redirects free and gets confused by thigns allocated by calloc
+ // so call the real free funtion through this warpper
+ real_free(this);
}
@@ -94,19 +96,19 @@
PhotCode *code = GetPhotcodebyCode(meas->photcode);
sprintf(sql,
- "INSERT INTO dvoDetection (imageID, ippDetectID, detectID, ippObjID, objID, flags, zp, zpErr, airMass, expTime, ra, dec_, raErr, decErr) VALUES (%d, %u, %lu, %lu, %lu, %u, %f, %f, %f, %f, %lf, %lf, %f %f)",
+ "INSERT INTO dvoDetection (imageID, ippDetectID, detectID, ippObjID, objID, flags, zp, zpErr, airMass, expTime, ra, dec_, raErr, decErr) VALUES (%d, %u, %lu, %lu, %lu, %u, %f, %f, %f, %f, %lf, %lf, %f, %f)",
image->imageID, // imageID
meas->detID, // ippDetectID
meas->extID, // detectID
(uint64_t)ave->catID*1000000000 + (uint64_t)ave->objID, // ippObjID
- dvoDetections[i].ave.extID, // objID
- meas->dbFlags, // flags
+ ave->extID, // objID
+ meas->dbFlags, // flags
code->C * 0.001 + code->K * (meas->airmass - 1) - meas->Mcal, // zp
- meas->dMcal, // zpErr
+ meas->dMcal, // zpErr
meas->airmass,
pow(10.0, 0.4 * meas->dt), // expTime
ave->R - meas->dR / 3600., // ra
ave->D - meas->dR / 3600., // dec
- meas->dXccd * meas->pltscale, // estimate of raErr
- meas->dYccd * meas->pltscale // estimate of decErr
+ meas->dXccd * 0.01 * fabs(meas->pltscale), // estimate of raErr
+ meas->dYccd * 0.01 * fabs(meas->pltscale) // estimate of decErr
);
Index: /branches/eam_branches/ipp-20121130/ippTools/configure.ac
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/configure.ac (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/configure.ac (revision 34839)
@@ -1,5 +1,5 @@
AC_PREREQ(2.61)
-AC_INIT([ipptools], [1.1.71], [ipp-support@ifa.hawaii.edu])
+AC_INIT([ipptools], [1.1.73], [ipp-support@ifa.hawaii.edu])
AC_CONFIG_SRCDIR([autogen.sh])
Index: /branches/eam_branches/ipp-20121130/ippTools/share/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/Makefile.am (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/Makefile.am (revision 34839)
@@ -88,4 +88,5 @@
chiptool_listrun.sql \
chiptool_pendingcleanupimfile.sql \
+ chiptool_pendingcleanupimfile_all.sql \
chiptool_pendingcleanuprun.sql \
chiptool_pendingimfile.sql \
@@ -172,4 +173,5 @@
difftool_pendingcleanuprun.sql \
difftool_pendingcleanupskyfile.sql \
+ difftool_pendingcleanupskyfile_all.sql \
difftool_revertdiffskyfile_delete.sql \
difftool_revertdiffskyfile_updated.sql \
@@ -391,4 +393,5 @@
stacktool_sassskyfile.sql \
stacktool_tosum.sql \
+ stacktool_tobkg.sql \
stacktool_tosummary.sql \
stacktool_addsummary.sql \
@@ -423,4 +426,5 @@
warptool_pendingcleanuprun.sql \
warptool_pendingcleanupskyfile.sql \
+ warptool_pendingcleanupskyfile_all.sql \
warptool_revertoverlap.sql \
warptool_revertwarped_delete.sql \
Index: /branches/eam_branches/ipp-20121130/ippTools/share/camtool_pendingcleanuprun.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/camtool_pendingcleanuprun.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/camtool_pendingcleanuprun.sql (revision 34839)
@@ -2,4 +2,6 @@
camRun.cam_id,
rawExp.camera,
+ rawExp.exp_tag,
+ camRun.workdir,
camRun.state
FROM camRun
Index: /branches/eam_branches/ipp-20121130/ippTools/share/chiptool_pendingcleanupimfile_all.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/chiptool_pendingcleanupimfile_all.sql (revision 34839)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/chiptool_pendingcleanupimfile_all.sql (revision 34839)
@@ -0,0 +1,16 @@
+-- list of chipProcessedImfiles for runs in a goto_cleaned like state without regard to
+-- data_state or quality. Used for confirming that a run is really in the cleaned state
+SELECT
+ chipProcessedImfile.*,
+ chipRun.state,
+ chipRun.workdir,
+ chipRun.label,
+ chipRun.reduction,
+ chipRun.expgroup,
+ chipRun.dvodb,
+ chipRun.tess_id,
+ chipRun.end_stage
+FROM chipRun
+JOIN chipProcessedImfile
+ USING(chip_id)
+WHERE ( chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_scrubbed' OR chipRun.state = 'goto_purged' )
Index: /branches/eam_branches/ipp-20121130/ippTools/share/chiptool_pendingcleanuprun.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/chiptool_pendingcleanuprun.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/chiptool_pendingcleanuprun.sql (revision 34839)
@@ -1,6 +1,8 @@
-SELECT
+SELECT DISTINCT
chipRun.chip_id,
rawExp.camera,
- chipRun.state
+ rawExp.exp_tag,
+ chipRun.state,
+ chipRun.workdir
FROM chipRun
JOIN rawExp
Index: /branches/eam_branches/ipp-20121130/ippTools/share/difftool_pendingcleanuprun.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/difftool_pendingcleanuprun.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/difftool_pendingcleanuprun.sql (revision 34839)
@@ -6,4 +6,6 @@
rawExp.camera,
diffRun.state,
+ diffRun.workdir,
+ diffRun.tess_id,
diffRun.label
FROM diffRun
@@ -35,4 +37,6 @@
rawExp.camera,
diffRun.state,
+ diffRun.workdir,
+ diffRun.tess_id,
diffRun.label
FROM diffRun
@@ -64,4 +68,6 @@
rawExp.camera,
diffRun.state,
+ diffRun.workdir,
+ diffRun.tess_id,
diffRun.label
FROM diffRun
@@ -93,4 +99,6 @@
rawExp.camera,
diffRun.state,
+ diffRun.workdir,
+ diffRun.tess_id,
diffRun.label
FROM diffRun
Index: /branches/eam_branches/ipp-20121130/ippTools/share/difftool_pendingcleanupskyfile_all.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/difftool_pendingcleanupskyfile_all.sql (revision 34839)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/difftool_pendingcleanupskyfile_all.sql (revision 34839)
@@ -0,0 +1,10 @@
+SELECT
+ diffSkyfile.*,
+ diffRun.state,
+ diffRun.workdir,
+ diffRun.dvodb,
+ diffRun.tess_id
+FROM diffRun
+JOIN diffSkyfile
+ USING(diff_id)
+WHERE ( diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged')
Index: /branches/eam_branches/ipp-20121130/ippTools/share/pstamptool_revertdependent.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/pstamptool_revertdependent.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/pstamptool_revertdependent.sql (revision 34839)
@@ -4,4 +4,6 @@
SET pstampDependent.fault = 0
-- fault count hook %s
-WHERE pstampDependent.state = 'new'
+WHERE pstampRequest.state ='run'
+ AND pstampJob.state ='run'
+ AND pstampDependent.state = 'new'
AND (pstampDependent.fault > 0)
Index: /branches/eam_branches/ipp-20121130/ippTools/share/pxadmin_create_tables.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/pxadmin_create_tables.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/pxadmin_create_tables.sql (revision 34839)
@@ -2119,4 +2119,5 @@
n_detections INT,
n_extended INT,
+ n_forced INT,
zpt_obs FLOAT,
zpt_stdev FLOAT,
Index: /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_inputskyfile.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_inputskyfile.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_inputskyfile.sql (revision 34839)
@@ -4,4 +4,5 @@
rawExp.exp_id,
rawExp.exp_name,
+ rawExp.exp_time,
rawExp.object,
rawExp.dateobs,
Index: /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_pendingcleanuprun.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_pendingcleanuprun.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_pendingcleanuprun.sql (revision 34839)
@@ -8,6 +8,8 @@
stackRun.stack_id,
rawExp.camera,
- stackRun.state
+ stackRun.state,
+ stackSumSkyfile.path_base
FROM stackRun
+JOIN stackSumSkyfile using(stack_id)
JOIN stackInputSkyfile
USING(stack_id)
Index: /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_tobkg.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_tobkg.sql (revision 34839)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/stacktool_tobkg.sql (revision 34839)
@@ -0,0 +1,29 @@
+SELECT
+ stackRun.stack_id,
+ stackRun.tess_id,
+ stackRun.skycell_id,
+ stackRun.workdir,
+ stackRun.reduction,
+ stackRun.label,
+ stackRun.state,
+ stackSumSkyfile.path_base,
+ rawExp.camera,
+ IFNULL(Label.priority, 10000) AS priority
+FROM stackRun
+JOIN stackInputSkyfile USING(stack_id)
+JOIN warpRun USING(warp_id)
+JOIN warpSkyfile USING(warp_id,skycell_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+LEFT JOIN stackSumSkyfile USING(stack_id)
+LEFT JOIN Label ON Label.label = stackRun.label
+WHERE
+ ((stackRun.state = 'full' AND stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0 AND stackSumSkyfile.background_model != 1))
+ AND (Label.active OR Label.active IS NULL)
+ -- WHERE hook %s
+GROUP BY stack_id
+HAVING (SUM(IF(warpRun.state = 'full', 1, 0)) = COUNT(stackInputSkyfile.warp_id) AND
+ SUM(IF(warpSkyfile.background_model = 1, 1, 0)) = COUNT(stackInputSkyfile.warp_id))
+
Index: /branches/eam_branches/ipp-20121130/ippTools/share/staticskytool_defineskycalrun.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/staticskytool_defineskycalrun.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/staticskytool_defineskycalrun.sql (revision 34839)
@@ -2,4 +2,6 @@
sky_id,
stack_id,
+ skycell_id,
+ filter,
staticskyRun.workdir,
staticskyRun.label,
@@ -11,4 +13,5 @@
JOIN stackRun USING(stack_id)
JOIN stackSumSkyfile USING(stack_id)
+ JOIN skycell USING(tess_id, skycell_id)
LEFT JOIN skycalRun USING(sky_id, stack_id)
WHERE staticskyRun.state = 'full'
Index: /branches/eam_branches/ipp-20121130/ippTools/share/staticskytool_pendingskycalrun.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/staticskytool_pendingskycalrun.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/staticskytool_pendingskycalrun.sql (revision 34839)
@@ -6,5 +6,5 @@
staticskyResult.path_base,
rawExp.camera,
- 0 AS singlefilter
+ staticskyResult.num_inputs = 1 AS singlefilter
FROM skycalRun
JOIN staticskyRun USING (sky_id)
Index: /branches/eam_branches/ipp-20121130/ippTools/share/warptool_pendingcleanuprun.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/warptool_pendingcleanuprun.sql (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/warptool_pendingcleanuprun.sql (revision 34839)
@@ -1,5 +1,7 @@
SELECT
warpRun.warp_id,
+ warpRun.workdir,
rawExp.camera,
+ rawExp.exp_tag,
warpRun.state
FROM warpRun
Index: /branches/eam_branches/ipp-20121130/ippTools/share/warptool_pendingcleanupskyfile_all.sql
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/share/warptool_pendingcleanupskyfile_all.sql (revision 34839)
+++ /branches/eam_branches/ipp-20121130/ippTools/share/warptool_pendingcleanupskyfile_all.sql (revision 34839)
@@ -0,0 +1,11 @@
+SELECT
+ warpSkyfile.*,
+ warpRun.state,
+ warpRun.workdir,
+ warpRun.label,
+ warpRun.dvodb,
+ warpRun.end_stage
+FROM warpRun
+JOIN warpSkyfile
+ USING(warp_id)
+WHERE (warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_scrubbed' OR warpRun.state = 'goto_purged')
Index: /branches/eam_branches/ipp-20121130/ippTools/src/chiptool.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/chiptool.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/chiptool.c (revision 34839)
@@ -1311,4 +1311,5 @@
PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false);
PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+ PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
@@ -1319,5 +1320,6 @@
pxAddLabelSearchArgs (config, where, "-label", "label", "==");
- psString query = pxDataGet("chiptool_pendingcleanupimfile.sql");
+ char *sql_file = all ? "chiptool_pendingcleanupimfile_all.sql" : "chiptool_pendingcleanupimfile.sql";
+ psString query = pxDataGet(sql_file);
if (!query) {
psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
Index: /branches/eam_branches/ipp-20121130/ippTools/src/chiptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/chiptoolConfig.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/chiptoolConfig.c (revision 34839)
@@ -296,4 +296,5 @@
psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0);
psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL);
+ psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-all", 0, "list all components regardless of data_state", false);
psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
Index: /branches/eam_branches/ipp-20121130/ippTools/src/difftool.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/difftool.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/difftool.c (revision 34839)
@@ -2561,4 +2561,5 @@
PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false);
+ PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
@@ -2570,5 +2571,6 @@
pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
- psString query = pxDataGet("difftool_pendingcleanupskyfile.sql");
+ char * sql_file = all ? "difftool_pendingcleanupskyfile_all.sql": "difftool_pendingcleanupskyfile.sql" ;
+ psString query = pxDataGet(sql_file);
if (!query) {
psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -2740,17 +2742,35 @@
PS_ASSERT_PTR_NON_NULL(config, false);
- PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
- PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
-
- psMetadata *where = psMetadataAlloc();
- PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
- PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
-
- if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) {
- psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+ PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
+
+ if (!state) {
+ psMetadata *where = psMetadataAlloc();
+ PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
+ PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
+
+ PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
+ PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
+
+ if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) {
+ psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+ psFree (where);
+ return false;
+ }
psFree (where);
- return false;
- }
- psFree (where);
+ } else {
+ if (strcmp(state,"error_cleaned") == 0) {
+ change_skyfile_data_state(config,"error_cleaned", "goto_cleaned");
+ }
+ else if (strcmp(state, "error_scrubbed") == 0) {
+ change_skyfile_data_state(config,"error_scrubbed", "goto_scrubbed");
+ }
+ else if (strcmp(state, "error_purged") == 0) {
+ change_skyfile_data_state(config,"error_purged", "goto_purged");
+ }
+ else {
+ psError(PS_ERR_UNKNOWN, false, "unhandled state given");
+ return(false);
+ }
+ }
return true;
Index: /branches/eam_branches/ipp-20121130/ippTools/src/difftoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/difftoolConfig.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/difftoolConfig.c (revision 34839)
@@ -395,4 +395,5 @@
psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0);
+ psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all", 0, "list all skycells regardless of data_state", false);
psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
@@ -411,4 +412,5 @@
psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+
// -updatediffskyfile
psMetadata *updatediffskyfileArgs = psMetadataAlloc();
@@ -416,5 +418,6 @@
psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0);
- psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0);
+ psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0);
+ psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set data_state state", NULL);
// -tocleanedskyfile
Index: /branches/eam_branches/ipp-20121130/ippTools/src/stacktool.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/stacktool.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/stacktool.c (revision 34839)
@@ -37,4 +37,5 @@
static bool inputskyfileMode(pxConfig *config);
static bool tosumMode(pxConfig *config);
+static bool tobkgMode(pxConfig *config);
static bool addsumskyfileMode(pxConfig *config);
static bool sumskyfileMode(pxConfig *config);
@@ -76,4 +77,5 @@
MODECASE(STACKTOOL_MODE_INPUTSKYFILE, inputskyfileMode);
MODECASE(STACKTOOL_MODE_TOSUM, tosumMode);
+ MODECASE(STACKTOOL_MODE_TOBKG, tobkgMode);
MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileMode);
MODECASE(STACKTOOL_MODE_SUMSKYFILE, sumskyfileMode);
@@ -952,4 +954,79 @@
// negative simple so the default is true
if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) {
+ psError(PS_ERR_UNKNOWN, false, "failed to print array");
+ psFree(output);
+ return false;
+ }
+ }
+
+ psFree(output);
+
+ return true;
+}
+
+static bool tobkgMode(pxConfig *config)
+{
+ PS_ASSERT_PTR_NON_NULL(config, false);
+
+ psMetadata *where = psMetadataAlloc();
+ PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
+ pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
+
+ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+ psString query = pxDataGet("stacktool_tobkg.sql");
+ if (!query) {
+ psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+ return false;
+ }
+
+ psString whereClause = psStringCopy(""); // WHERE conditions to add
+ if (psListLength(where->list)) {
+ psString new = psDBGenerateWhereConditionSQL(where, NULL);
+ psStringAppend(&whereClause, "\nAND %s", new);
+ psFree(new);
+ }
+ psFree(where);
+
+ psStringAppend(&query, "\nORDER by priority DESC, stack_id");
+
+ // treat limit == 0 as "no limit"
+ if (limit) {
+ psString limitString = psDBGenerateLimitSQL(limit);
+ psStringAppend(&query, " %s", limitString);
+ psFree(limitString);
+ }
+
+ if (!p_psDBRunQueryF(config->dbh, query, whereClause)) {
+ psError(PS_ERR_UNKNOWN, false, "database error");
+ psFree(query);
+ return false;
+ }
+ psFree(query);
+
+ psArray *output = p_psDBFetchResult(config->dbh);
+ if (!output) {
+ psErrorCode err = psErrorCodeLast();
+ switch (err) {
+ case PS_ERR_DB_CLIENT:
+ psError(PXTOOLS_ERR_SYS, false, "database error");
+ case PS_ERR_DB_SERVER:
+ psError(PXTOOLS_ERR_PROG, false, "database error");
+ default:
+ psError(PXTOOLS_ERR_PROG, false, "unknown error");
+ }
+
+ return false;
+ }
+ if (!psArrayLength(output)) {
+ psTrace("stacktool", PS_LOG_INFO, "no rows found");
+ psFree(output);
+ return true;
+ }
+
+ if (psArrayLength(output)) {
+ // negative simple so the default is true
+ if (!ippdbPrintMetadatas(stdout, output, "stackBkgSkyfile", !simple)) {
psError(PS_ERR_UNKNOWN, false, "failed to print array");
psFree(output);
@@ -1681,12 +1758,26 @@
PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
+ PXOPT_LOOKUP_S16(background_model, config->args, "-set_background_model", false, false);
psMetadata *where = psMetadataAlloc();
PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
- if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault, quality)) {
+ if (background_model) {
+ psMetadata *values = psMetadataAlloc();
+ PXOPT_COPY_S16(config->args, values, "-set_background_model", "background_model", "==");
+ long rows = psDBUpdateRows(config->dbh,"stackSumSkyfile", where, values);
+ psFree(values);
+ if (!rows) {
+ // This maybe should rollback and error if rows != 1
+ psError(PS_ERR_UNKNOWN, true, "no rows changed");
+ return false;
+ }
+ }
+ else {
+ if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault, quality)) {
psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
psFree (where);
return false;
+ }
}
psFree (where);
Index: /branches/eam_branches/ipp-20121130/ippTools/src/stacktool.h
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/stacktool.h (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/stacktool.h (revision 34839)
@@ -31,4 +31,5 @@
STACKTOOL_MODE_INPUTSKYFILE,
STACKTOOL_MODE_TOSUM,
+ STACKTOOL_MODE_TOBKG,
STACKTOOL_MODE_ADDSUMSKYFILE,
STACKTOOL_MODE_SUMSKYFILE,
Index: /branches/eam_branches/ipp-20121130/ippTools/src/stacktoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/stacktoolConfig.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/stacktoolConfig.c (revision 34839)
@@ -150,4 +150,11 @@
psMetadataAddBool(tosumArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+ // -tobkg
+ psMetadata *tobkgArgs = psMetadataAlloc();
+ psMetadataAddS64(tobkgArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0);
+ psMetadataAddStr(tobkgArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
+ psMetadataAddU64(tobkgArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+ psMetadataAddBool(tobkgArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
// -addsumskyfile
psMetadata *addsumskyfileArgs = psMetadataAlloc();
@@ -280,5 +287,6 @@
psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0);
psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0);
-
+ psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-set_background_model", 0, "set background model", 0);
+
// -exportrun
psMetadata *exportrunArgs = psMetadataAlloc();
@@ -304,4 +312,5 @@
PXOPT_ADD_MODE("-inputskyfile", "", STACKTOOL_MODE_INPUTSKYFILE, inputskyfileArgs);
PXOPT_ADD_MODE("-tosum", "", STACKTOOL_MODE_TOSUM, tosumArgs);
+ PXOPT_ADD_MODE("-tobkg", "", STACKTOOL_MODE_TOBKG, tobkgArgs);
PXOPT_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs);
PXOPT_ADD_MODE("-sumskyfile", "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs);
Index: /branches/eam_branches/ipp-20121130/ippTools/src/staticskytool.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/staticskytool.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/staticskytool.c (revision 34839)
@@ -132,14 +132,4 @@
pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE");
pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE");
-#ifdef notdef
- PXOPT_COPY_F32(config->args, whereMD, "-select_glat_min", "skycell.glat", ">=");
- PXOPT_COPY_F32(config->args, whereMD, "-select_glat_max", "skycell.glat", "<=");
- PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_min", "skycell.radeg/15", ">=");
- PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_max", "skycell.radeg/15", "<=");
- PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_min", "skycell.radeg", ">=");
- PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_max", "skycell.radeg", "<=");
- PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_min", "skycell.decdeg", ">=");
- PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_max", "skycell.decdeg", "<=");
-#endif
if (!pxskycellAddWhere(config, whereMD)) {
psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
@@ -686,5 +676,5 @@
psStringAppend(&query, "\nGROUP BY sky_id");
if (num_filters) {
- psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters);
+ psStringAppend(&query, "\nHAVING COUNT(filter) = %d", num_filters);
}
@@ -1049,4 +1039,8 @@
pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "staticskyRun.data_group", "LIKE");
pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE");
+ if (!pxskycellAddWhere(config, whereMD)) {
+ psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
+ return false;
+ }
PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
@@ -1128,4 +1122,5 @@
psString sky_label = psMetadataLookupStr(&status, row, "label");
psString sky_data_group = psMetadataLookupStr(&status, row, "data_group");
+ psString sky_dist_group = psMetadataLookupStr(&status, row, "dist_group");
// create a staticskyRun
@@ -1138,5 +1133,5 @@
label ? label : sky_label,
data_group ? data_group : sky_data_group,
- dist_group,
+ dist_group ? dist_group : sky_dist_group,
reduction,
registered,
@@ -1159,7 +1154,13 @@
psMetadata *where = psMetadataAlloc();
- PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "==");
- PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
- PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
+ PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "==");
+ PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "==");
+ PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
+ PXOPT_COPY_STR(config->args, where, "-label", "skycalRun.label", "==");
+ PXOPT_COPY_STR(config->args, where, "-state", "skycalRun.state", "==");
+ if (!pxskycellAddWhere(config, where)) {
+ psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
+ return false;
+ }
if (!psListLength(where->list)) {
psFree(where);
@@ -1168,5 +1169,5 @@
}
- psString query = psStringCopy("UPDATE skycalRun");
+ psString query = psStringCopy("UPDATE skycalRun JOIN stackRun using(stack_id) join skycell using(tess_id, skycell_id)");
// pxUpdateRun gets parameters from config->args and updates
@@ -1276,6 +1277,7 @@
PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
PXOPT_LOOKUP_S32(n_astrom, config->args, "-n_astrom", false, false);
- PXOPT_LOOKUP_S32(n_detections, config->args, "-n_detections", false, false);
- PXOPT_LOOKUP_S32(n_extended, config->args, "-n_extended", false, false);
+ PXOPT_LOOKUP_S32(n_detections, config->args,"-n_detections", false, false);
+ PXOPT_LOOKUP_S32(n_extended, config->args, "-n_extended", false, false);
+ PXOPT_LOOKUP_S32(n_forced, config->args, "-n_forced", false, false);
PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
@@ -1318,4 +1320,5 @@
n_detections,
n_extended,
+ n_forced,
zpt_obs,
zpt_stdev,
Index: /branches/eam_branches/ipp-20121130/ippTools/src/staticskytoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/staticskytoolConfig.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/staticskytoolConfig.c (revision 34839)
@@ -126,5 +126,5 @@
psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
- psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0);
+ psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs", 0);
pxskycellAddArguments(resultArgs);
psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
@@ -166,4 +166,5 @@
psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL);
psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL);
+ pxskycellAddArguments(defineskycalrunArgs);
psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL);
psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL);
@@ -181,6 +182,9 @@
psMetadata *updateskycalrunArgs = psMetadataAlloc();
psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0);
+ psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky ID", 0);
+ psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0);
psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
+ pxskycellAddArguments(updateskycalrunArgs);
psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL);
psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);
@@ -215,4 +219,5 @@
psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_detections", 0, "define number of detections", 0);
psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_extended", 0, "define number of extended detections", 0);
+ psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_forced", 0, "define number of forced detections", 0);
psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL);
Index: /branches/eam_branches/ipp-20121130/ippTools/src/warptool.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/warptool.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/warptool.c (revision 34839)
@@ -1838,4 +1838,5 @@
PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+ PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
psMetadata *where = psMetadataAlloc();
@@ -1843,5 +1844,6 @@
pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
- psString query = pxDataGet("warptool_pendingcleanupskyfile.sql");
+ char * sql_file = all ? "warptool_pendingcleanupskyfile_all.sql" : "warptool_pendingcleanupskyfile.sql";
+ psString query = pxDataGet(sql_file);
if (!query) {
psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -2057,4 +2059,5 @@
PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
+
if (magicked) {
psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked);
@@ -2063,4 +2066,8 @@
if (quality) {
psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.quality = %"PRId16, quality);
+ }
+ PXOPT_LOOKUP_S16(background_model, config->args, "-set_background_model", false, false);
+ if (background_model) {
+ psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.background_model = %"PRId16, background_model);
}
} else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
@@ -2123,6 +2130,22 @@
// warp_id, skycell_id, fault are required
PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
-
- if (!state) {
+ PXOPT_LOOKUP_S16(background_model, config->args, "-set_background_model", false, false);
+ if (background_model) {
+ // CZW 2012-12-06: I'm unclear why we don't use this form for all updates?
+ psMetadata *where = psMetadataAlloc();
+ psMetadata *values = psMetadataAlloc();
+ PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
+ PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
+ PXOPT_COPY_S16(config->args, values, "-set_background_model", "background_model", "==");
+ long rows = psDBUpdateRows(config->dbh,"warpSkyfile", where, values);
+ psFree(values);
+ psFree(where);
+ if (!rows) {
+ // This maybe should rollback and error if rows != 1
+ psError(PS_ERR_UNKNOWN, true, "no rows changed");
+ return false;
+ }
+ }
+ else if (!state) {
psMetadata *where = psMetadataAlloc();
PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
Index: /branches/eam_branches/ipp-20121130/ippTools/src/warptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20121130/ippTools/src/warptoolConfig.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/ippTools/src/warptoolConfig.c (revision 34839)
@@ -364,4 +364,5 @@
psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID (required)", 0);
+ psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all", 0, "list all skycells regardless of data_state", false);
psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
@@ -409,5 +410,5 @@
psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-set_quality", 0,"new quality value", 0);
psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", 0);
-
+ psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-set_background_model", 0, "set the background_model value", 0);
// -exportrun
psMetadata *exportrunArgs = psMetadataAlloc();
Index: /branches/eam_branches/ipp-20121130/magic/censorObjects/src/censorLoop.c
===================================================================
--- /branches/eam_branches/ipp-20121130/magic/censorObjects/src/censorLoop.c (revision 34838)
+++ /branches/eam_branches/ipp-20121130/magic/censorObjects/src/censorLoop.c (revision 34839)
@@ -29,6 +29,4 @@
-
-
// select the current recipe
psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, CENSOR_RECIPE);
@@ -58,4 +56,6 @@
// check on the mask file at this level if possible
badMaskFile = censorBadFileByView(badMaskFile, inMask, config, view);
+
+ bool checkInputOnly = psMetadataLookupBool(NULL, config->arguments, "CHECK.INPUT.ONLY");
// files associated with the science image
@@ -116,5 +116,7 @@
if (!readout->data_exists) { continue; }
- if (!censorReadout (readout, config, view, badMaskFile)) ESCAPE;
+ if (!checkInputOnly) {
+ if (!censorReadout (readout, config, view, badMaskFile)) ESCAPE;
+ }
if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;