Index: branches/eam_branches/ipp-20131211/dbconfig/changes.txt =================================================================== --- branches/eam_branches/ipp-20131211/dbconfig/changes.txt (revision 36479) +++ branches/eam_branches/ipp-20131211/dbconfig/changes.txt (revision 36480) @@ -2432,2 +2432,64 @@ UPDATE dbversion set schema_version = '1.1.77', updated= CURRENT_TIMESTAMP(); +-- + +CREATE TABLE fullForceRun ( + ff_id BIGINT NOT NULL AUTO_INCREMENT, + skycal_id BIGINT, + sources_path_base VARCHAR(255), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + note VARCHAR(255), + reduction VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(ff_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(skycal_id), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceInput ( + ff_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(ff_id, warp_id), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceResult ( + ff_id BIGINT, + warp_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id, warp_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceSummary ( + ff_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +UPDATE dbversion set schema_version = '1.1.78', updated= CURRENT_TIMESTAMP(); + Index: branches/eam_branches/ipp-20131211/dbconfig/ff.md =================================================================== --- branches/eam_branches/ipp-20131211/dbconfig/ff.md (revision 36480) +++ branches/eam_branches/ipp-20131211/dbconfig/ff.md (revision 36480) @@ -0,0 +1,41 @@ + +fullForceRun METADATA + ff_id S64 0 + skycal_id S64 0 + sources_path_base STR 255 + state STR 64 + workdir STR 255 + label STR 64 + data_group STR 64 + dist_group STR 64 + note STR 255 + reduction STR 64 + registered TAI NULL +END + +fullForceInput METADATA + ff_id S64 0 + warp_id S64 0 +END + +fullForceResult METADATA + ff_id S64 0 + warp_id S64 0 + path_base STR 255 + dtime_script F32 0.0 + quality S16 0 + hostname STR 64 + software_ver STR 16 + fault S16 0 +END + +fullForceSummary METADATA + ff_id S64 0 + path_base STR 255 + dtime_script F32 0.0 + quality S16 0 + hostname STR 64 + software_ver STR 16 + fault S16 0 +END + Index: branches/eam_branches/ipp-20131211/dbconfig/ipp.m4 =================================================================== --- branches/eam_branches/ipp-20131211/dbconfig/ipp.m4 (revision 36479) +++ branches/eam_branches/ipp-20131211/dbconfig/ipp.m4 (revision 36480) @@ -40,2 +40,3 @@ include(skycell.md) include(release.md) +include(ff.md) Index: branches/eam_branches/ipp-20131211/ippMonitor/czartool/czartool/Gpc1Db.pm =================================================================== --- branches/eam_branches/ipp-20131211/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 36479) +++ branches/eam_branches/ipp-20131211/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 36480) @@ -139,4 +139,9 @@ my $stateCol = $table.".state"; + my $extraState = ""; + if ($state eq 'new') { + $extraState = " OR $stateCol = 'update' "; + } + my $query = $self->{_db}->prepare(<execute; @@ -208,10 +213,12 @@ my $query = undef; + my $joinTable = undef; + my $id = undef; + if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;} + + # XXX: the "update" page is now obsolete. The standard processing page should now show + # counts in new and update state if ($state eq "update") { - if ($stage eq "dist") {return 0;} - my $joinTable = undef; - my $id = undef; - if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;} if ($stage eq "chip" || $stage eq "fake" || $stage eq "warp" || $stage eq "diff" || $stage eq "magicDS") { @@ -226,12 +233,11 @@ } else { - $query = $self->{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{Statement} . "\n"; $query->execute; Index: branches/eam_branches/ipp-20131211/ippScripts/Build.PL =================================================================== --- branches/eam_branches/ipp-20131211/ippScripts/Build.PL (revision 36479) +++ branches/eam_branches/ipp-20131211/ippScripts/Build.PL (revision 36480) @@ -133,4 +133,5 @@ scripts/queuestaticsky.pl scripts/psphot_fullforce_warp.pl + scripts/psphot_fullforce_summary.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: branches/eam_branches/ipp-20131211/ippScripts/scripts/dist_bundle.pl =================================================================== --- branches/eam_branches/ipp-20131211/ippScripts/scripts/dist_bundle.pl (revision 36479) +++ branches/eam_branches/ipp-20131211/ippScripts/scripts/dist_bundle.pl (revision 36480) @@ -50,8 +50,22 @@ 'PPSUB.INVERSE.MASK' => 'inv_mask', 'PPSUB.INVERSE.VARIANCE' => 'inv_variance' ); -my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image', + +my %stack_cleaned = ( 'PPSTACK.UNCONV' => 'image', + 'PPSTACK.UNCONV.MASK' => 'mask', + 'PPSTACK.UNCONV.VARIANCE' => 'variance', + 'PPSTACK.UNCONV.EXP' => 'exp', + 'PPSTACK.UNCONV.EXPNUM' => 'expnum', + 'PPSTACK.UNCONV.EXPWT' => 'expwt' + ); + +my %stack_convolved = ( 'PPSTACK.OUTPUT' => 'image', 'PPSTACK.OUTPUT.MASK' => 'mask', - 'PPSTACK.OUTPUT.VARIANCE' => 'variance' ); + 'PPSTACK.OUTPUT.VARIANCE' => 'variance', + 'PPSTACK.OUTPUT.EXP' => 'exp', + 'PPSTACK.OUTPUT.EXPNUM' => 'expnum', + 'PPSTACK.OUTPUT.EXPWT' => 'expwt' + ); my %empty_cleaned = (); + @@ -136,5 +150,6 @@ if (($stage ne 'raw') and ($stage ne 'fake') and ($stage ne 'stack_summary') and !$poor_quality) { # If the file list is empty it is an error because we should at least get a config dump file - # except for fake stage which doesn't do anything yet + # except for fake stage which doesn't do anything yet, stack_summary stage, and of course raw files have + # no config dump because they aren't processed &my_die("empty file list", $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list); } @@ -165,6 +180,14 @@ my $image_type = get_image_type($stage, $file_rule); + # skip useless empty trace files + next if ($file_rule =~ /TRACE/); + # if this is an image and we are building a clean bundle or if quality is bad skip this rule next if $image_type and ($clean or $poor_quality); + + if ($stage eq 'stack') { + # skip convolved stacks since they are deleted just after they are created now + next if $stack_convolved{$file_rule}; + } # if magic is required, don't ship jpegs or binned fits images @@ -180,6 +203,15 @@ my $file_name = $file->{name}; - my $base = basename($file_name); my $path = $ipprc->file_resolve($file_name); + + if (!$path and $file_rule =~ /LOG/) { + my $compressed_file_name = $file_name . '.bz2'; + my $compressed_path = $ipprc->file_resolve($compressed_file_name); + if ($compressed_path) { + $path = $compressed_path; + $file_name = $compressed_file_name; + } + } + if (!$path) { @@ -194,7 +226,7 @@ # XXX: perhaps only do this for stages where we know that this happens next if $file_rule =~ /STATS/; - # don't fail on these "non-essential files" + # don't fail if these "non-essential files" are missing + next if $file_rule =~ /LOG/; next if $file_rule =~ /TRACE/; - next if $file_rule =~ /LOG/; next if $file_rule =~ /BIN/; @@ -236,5 +268,5 @@ # create a symbolic link from the file in the nebulous repository # in the temporary directory - symlink $path, "$tmpdir/$base"; + symlink $path, "$tmpdir/" . basename($file_name); } } Index: branches/eam_branches/ipp-20131211/ippScripts/scripts/psphot_fullforce_summary.pl =================================================================== --- branches/eam_branches/ipp-20131211/ippScripts/scripts/psphot_fullforce_summary.pl (revision 36480) +++ branches/eam_branches/ipp-20131211/ippScripts/scripts/psphot_fullforce_summary.pl (revision 36480) @@ -0,0 +1,295 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $psphotFullForceSummary = can_run('psphotFullForceSummary') or (warn "Can't find psphotFullForceSummary" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($ff_id, $outroot, $reduction, $camera); +my ($dbname, $threads, $verbose, $save_temps, $no_update, $no_op, $redirect); + +GetOptions( + 'ff_id=s' => \$ff_id, + 'camera=s' => \$camera, # camera name of sources + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --ff_id --outroot --camera", + -exitval => 3, + ) + unless defined $ff_id, + and defined $camera + and defined $outroot; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $ff_id, $PS_EXIT_CONFIG_ERROR ); + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { + $neb = $ipprc->nebulous(); +} + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", + $ff_id, $PS_EXIT_SYS_ERROR ) if $redirect; + + +my ($listFile, $listName) = tempfile("/tmp/fullforce.summary.list.XXXX", UNLINK => !$save_temps ); + +{ + my $mdcParser = PS::IPP::Metadata::Config->new; + my $results; + { + my $command = "$fftool -result -ff_id $ff_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform fftool -result $error_code", + $ff_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $ff_id, $PS_EXIT_PROG_ERROR); + $results = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $ff_id, $PS_EXIT_PROG_ERROR); + } + + &my_die("result list is empty.", $ff_id, $PS_EXIT_SYS_ERROR) + if scalar @$results == 0; + + print $listFile "SOURCES MULTI\n"; + foreach my $result (@$results) { + my $cmf = $ipprc->filename('PSPHOT.FULLFORCE.OUTPUT', $result->{path_base}); + &my_die("Couldn't find input cmf: $cmf", $ff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($cmf); + print $listFile "SOURCES STR $cmf\n"; + } + close $listFile; +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'FULLFORCE_PSPHOT'); # Recipe to use for psphot +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", + $ff_id, $PS_EXIT_CONFIG_ERROR); +} + +# XXX: need to figure out whether this works or not +# We probably want to create a specific recipe that looks at the results +my $recipe_ppstats = 'WARPSTATS'; +my $doStats = 0; + +print "reduction: $reduction\n"; +print "recipe_psphot: $recipe_psphot\n"; + +my $dump_config = 1; + +# Get the output filenames +my $outputSources = prepare_output("PSPHOT.FULLFORCE.OUTPUT", $outroot, 1); +my $configuration = prepare_output("PSPHOT.SKY.CONFIG", $outroot, 1) if $dump_config; +my $outputStats = prepare_output("SKYCELL.STATS", $outroot, 1) if $doStats; +my $traceDest = prepare_output("TRACE.EXP", $outroot, 1); + +my $cmdflags = ""; + +# Perform psphotFullForceSummary +{ + my $command = "$psphotFullForceSummary $outroot"; + $command .= " -input $listName"; + $command .= " -threads $threads" if defined $threads; + if ($dump_config) { + $command .= " -dumpconfig $configuration"; + } + $command .= " -recipe PSPHOT $recipe_psphot"; + if ($doStats) { + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS $recipe_ppstats"; + } +# $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psphotFullForceSummary: $error_code", $ff_id, $error_code); + } + + # Stats: TODO + if ($doStats) { + check_output($outputStats, 1); + my $outputStatsReal = $ipprc->file_resolve($outputStats); + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $ff_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + } + my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + if (!$quality) { + check_output($outputSources, 1); + } + } else { + print "Not executing: $command\n"; + } +} + +# Add the result to the database +{ + my $command = "$fftool -ff_id $ff_id"; + $command .= " -addsummary -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform fftool -addwarped" ; + &my_die("$err_message: $error_code", $ff_id, $error_code); + } + } else { + print "Not executing $command\n"; + } +} + +exit 0; + + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $ff_id, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $ff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $ff_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $ff_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $ff_id, $PS_EXIT_SYS_ERROR); + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $ff_id = shift; # full force run identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $ff_id) { + my $command = "$fftool -ff_id $ff_id -fault $exit_code"; + $command .= " -addsummary"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + run(command => $command, verbose => $verbose); + } else { + print "not executing $command\n"; + } + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: branches/eam_branches/ipp-20131211/ippScripts/scripts/psphot_fullforce_warp.pl =================================================================== --- branches/eam_branches/ipp-20131211/ippScripts/scripts/psphot_fullforce_warp.pl (revision 36479) +++ branches/eam_branches/ipp-20131211/ippScripts/scripts/psphot_fullforce_warp.pl (revision 36480) @@ -31,6 +31,5 @@ my $psphotFullForce = can_run('psphotFullForce') or (warn "Can't find psphotFullForce" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); -# XXX: fftool is yet to be written -my $fftool = "fftool"; # can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); +my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -38,14 +37,14 @@ } -my ($ffw_id, $warp_id, $skycell_id, $path_base, $sourceroot, $camera); +my ($ff_id, $warp_id, $skycell_id, $path_base, $sourceroot, $camera); my ($outroot, $reduction); my ($dbname, $threads, $verbose, $no_update, $no_op, $redirect); GetOptions( - 'ffw_id=s' => \$ffw_id, + 'ff_id=s' => \$ff_id, 'warp_id=s' => \$warp_id, # warp identifier 'skycell_id=s' => \$skycell_id,# Skycell identifier 'warp_path_base=s' => \$path_base, # path_base of the warp skycell - 'sourceroot=s' => \$sourceroot,# path_base of sources + 'sources_path_base=s' => \$sourceroot,# path_base of sources 'camera=s' => \$camera, # camera name of sources 'dbname|d=s' => \$dbname, # Database name @@ -61,17 +60,15 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --ffw_id --sourceroot --skycell_id --warp_path_base --outroot --camera", + -msg => "Required options: --ff_id --warp_id --sourceroot --skycell_id --warp_path_base --outroot --camera", -exitval => 3, - ) unless defined $ffw_id, + ) unless defined $ff_id, and defined $sourceroot - and (defined $path_base or defined $warp_id) # if we don't have warp's path_base we need warp_id + and defined $path_base + and defined $warp_id and defined $skycell_id and defined $camera and defined $outroot; -# XXX: fftool is not ready to run commands that update the database -$no_update = 1; - -my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $ffw_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $ff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); my $neb; @@ -84,7 +81,8 @@ $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", - $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; - -if (!$path_base) { + $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +if (0) { +# if (!$path_base) { # If path_base is not supplied, look it up in the database. my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -98,14 +96,14 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform warptool -warpskyfile -inputskyfile: $error_code", - $ffw_id, $skycell_id, $error_code); + $ff_id, $warp_id, $skycell_id, $error_code); } my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $ffw_id, $skycell_id, $PS_EXIT_PROG_ERROR); + &my_die("Unable to parse metadata config doc", $ff_id, $warp_id, $skycell_id, $PS_EXIT_PROG_ERROR); $files = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $ffw_id, $skycell_id, $PS_EXIT_PROG_ERROR); - } - - &my_die("Input list does not contain exactly one elements", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) + &my_die("Unable to parse metadata list", $ff_id, $warp_id, $skycell_id, $PS_EXIT_PROG_ERROR); + } + + &my_die("Input list does not contain exactly one elements", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless scalar @$files == 1; @@ -113,5 +111,5 @@ $path_base = $warp->{path_base}; - &my_die("Couldn't find input path in warptool output", $ffw_id, $skycell_id, $PS_EXIT_UNKNOWN_ERROR) + &my_die("Couldn't find input path in warptool output", $ff_id, $warp_id, $skycell_id, $PS_EXIT_UNKNOWN_ERROR) unless defined $path_base; @@ -123,5 +121,5 @@ unless ($recipe_psphot) { &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", - $ffw_id, $skycell_id, $PS_EXIT_CONFIG_ERROR); + $ff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR); } @@ -134,8 +132,12 @@ print "recipe_psphot: $recipe_psphot\n"; +# use psf measured on input warp +# XXX: get this from recipe +my $useWarpPSF = 0; + my $input = $ipprc->filename('PSWARP.OUTPUT', $path_base); my $inputMask = $ipprc->filename('PSWARP.OUTPUT.MASK', $path_base); my $inputVariance = $ipprc->filename('PSWARP.OUTPUT.VARIANCE', $path_base); -my $inputPSF = $ipprc->filename('PSPHOT.PSF.SKY.SAVE', $path_base); +my $inputPSF = $useWarpPSF ? $ipprc->filename('PSPHOT.PSF.SKY.SAVE', $path_base) : ""; my $inputSources = $ipprc->filename('PSPHOT.OUTPUT.CFF', $sourceroot); @@ -144,15 +146,14 @@ print "inputMask: $inputMask\n"; print "inputVariance: $inputVariance\n"; - # print "inputPath: $path_base\n"; - print "inputPSF: $inputPSF\n"; + print "inputPSF: $inputPSF\n" if $inputPSF; print "inputSources: $inputSources\n"; } # check that the inputs exist (and have non-zero size) -&my_die("Couldn't find input: $input", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); -&my_die("Couldn't find input: $inputMask", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); -&my_die("Couldn't find input: $inputVariance", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance); -&my_die("Couldn't find input: $inputPSF", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputPSF); -&my_die("Couldn't find input: $inputSources", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputSources); +&my_die("Couldn't find input: $input", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); +&my_die("Couldn't find input: $inputMask", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputVariance", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance); +&my_die("Couldn't find input: $inputPSF", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) if ($inputPSF && !$ipprc->file_exists($inputPSF)); +&my_die("Couldn't find input: $inputSources", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputSources); my $dump_config = 1; @@ -193,5 +194,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppSub: $error_code", $ffw_id, $skycell_id, $error_code); + &my_die("Unable to perform ppSub: $error_code", $ff_id, $warp_id, $skycell_id, $error_code); } @@ -207,5 +208,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppStatsFromMetadata: $error_code", $ffw_id, $skycell_id, $error_code); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $ff_id, $warp_id, $skycell_id, $error_code); } foreach my $line (@$stdout_buf) { @@ -226,6 +227,6 @@ # Add the result to the database { - my $command = "$fftool -ffw_id $ffw_id -skycell_id $skycell_id"; - $command .= " -addwarped -path_base $outroot"; + my $command = "$fftool -ff_id $ff_id -warp_id $warp_id"; + $command .= " -addresult -path_base $outroot"; $command .= " $cmdflags"; $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); @@ -239,5 +240,5 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); my $err_message = "Unable to perform fftool -addwarped" ; - &my_die("$err_message: $error_code", $ffw_id, $skycell_id, $error_code); + &my_die("$err_message: $error_code", $ff_id, $warp_id, $skycell_id, $error_code); } } else { @@ -262,5 +263,5 @@ my $error; my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error) - or &my_die("failed to prepare output file for: $filerule", $ffw_id, $skycell_id, $error); + or &my_die("failed to prepare output file for: $filerule", $ff_id, $warp_id, $skycell_id, $error); return $output; } @@ -275,15 +276,15 @@ } - &my_die("Couldn't find expected output file: $file", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + &my_die("Couldn't find expected output file: $file", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); # Funpack to confirm we've really made things correctly my $diskfile = $ipprc->file_resolve($file); if ($diskfile =~ /fits/) { - my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR); + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); my $check_command = "$funpack -S $diskfile > /dev/null"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $check_command, verbose => $verbose); if (!$success) { - &my_die("Output file not a valid fits file: $file", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR); + &my_die("Output file not a valid fits file: $file", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); } } @@ -291,5 +292,5 @@ if ($replicate and $neb) { - $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $ffw_id, $skycell_id, $PS_EXIT_SYS_ERROR); + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $ff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); } } @@ -299,5 +300,6 @@ { my $msg = shift; # Warning message on die - my $ffw_id = shift; # full force warp identifier + my $ff_id = shift; # full force run identifier + my $warp_id = shift; # full force warp id my $skycell_id = shift; # Skycell identifier my $exit_code = shift; # Exit code to add @@ -306,7 +308,7 @@ warn($msg); - if (defined $ffw_id and defined $skycell_id) { - my $command = "$fftool -ffw_id $ffw_id -skycell_id $skycell_id -fault $exit_code"; - $command .= " -addffskyfile"; + if (defined $ff_id and defined $skycell_id) { + my $command = "$fftool -ff_id $ff_id -warp_id $warp_id -fault $exit_code"; + $command .= " -addresult"; $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); $command .= " -hostname $host" if defined $host; Index: branches/eam_branches/ipp-20131211/ippScripts/scripts/queuestaticsky.pl =================================================================== --- branches/eam_branches/ipp-20131211/ippScripts/scripts/queuestaticsky.pl (revision 36479) +++ branches/eam_branches/ipp-20131211/ippScripts/scripts/queuestaticsky.pl (revision 36480) @@ -33,4 +33,6 @@ }; +# don't update the database if we are pretending +$no_update = 1 if $pretend; my $missing_tools; @@ -113,4 +115,5 @@ } } + last if $pretend; # only do the first (full) set if we are in pretend mode } { Index: branches/eam_branches/ipp-20131211/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/eam_branches/ipp-20131211/ippScripts/scripts/stack_skycell.pl (revision 36479) +++ branches/eam_branches/ipp-20131211/ippScripts/scripts/stack_skycell.pl (revision 36480) @@ -40,5 +40,5 @@ } -my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps, $delete_convolved_images); GetOptions( 'stack_id|d=s' => \$stack_id, # Stack identifier @@ -55,4 +55,5 @@ 'redirect-output' => \$redirect, 'save-temps' => \$save_temps, # Save temporary files? + 'delete-convolved' => \$delete_convolved_images, ) or pod2usage( 2 ); @@ -92,4 +93,5 @@ ); + my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration $| = 1; @@ -360,5 +362,25 @@ if (!$quality) { + + check_outputs(\@outputFiles, $replicate_outputs); + + if (($convolve)&&($delete_convolved_images)) { # We made convolved products, but do not wish to keep them anymore + my @products_to_clear = ('PPSTACK.OUTPUT', + 'PPSTACK.OUTPUT.MASK', + 'PPSTACK.OUTPUT.VARIANCE', + 'PPSTACK.OUTPUT.EXP', + 'PPSTACK.OUTPUT.EXPNUM', + 'PPSTACK.OUTPUT.EXPWT'); + foreach my $product (@products_to_clear) { + my $file = $ipprc->filename($product,$outroot,$skycell_id); + print "Deleting unwanted convolved product: $file\n"; + my $error= $ipprc->kill_file($file); + if ($error_code) { + print "Failed to delete unwanted convolved product: $error_code\n"; + } + } + } + } Index: branches/eam_branches/ipp-20131211/ippScripts/scripts/staticsky.pl =================================================================== --- branches/eam_branches/ipp-20131211/ippScripts/scripts/staticsky.pl (revision 36479) +++ branches/eam_branches/ipp-20131211/ippScripts/scripts/staticsky.pl (revision 36480) @@ -138,4 +138,6 @@ my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); + my $needConvolvedImages = 0; + foreach my $file (@$files) { print $listFile "INPUT METADATA\n"; @@ -146,8 +148,11 @@ my $stack_id = $file->{stack_id}; - my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name - my $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name - my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name - my $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM", $path_base ); # Expnum name + my ($imageCnv, $maskCnv, $weightCnv, $expnumCnv); + if ($needConvolvedImages) { + $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name + $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name + $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name + $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM", $path_base ); # Expnum name + } my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $path_base ); # Image name @@ -166,9 +171,11 @@ &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw"); &my_die("Couldn't find input: $expnumRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumRaw"); - &my_die("Couldn't find input: $imageCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); - &my_die("Couldn't find input: $maskCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); - &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); - &my_die("Couldn't find input: $expnumCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv"); - &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); + if ($needConvolvedImages) { + &my_die("Couldn't find input: $imageCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); + &my_die("Couldn't find input: $maskCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); + &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); + &my_die("Couldn't find input: $expnumCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv"); + &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); + } &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); @@ -178,12 +185,12 @@ print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; print $listFile " RAW:EXPNUM STR " . $expnumRaw . "\n"; - - print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; - print $listFile " CNV:MASK STR " . $maskCnv . "\n"; - print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; - print $listFile " CNV:EXPNUM STR " . $expnumCnv . "\n"; - print $listFile " CNV:PSF STR " . $psfCnv . "\n"; - print $listFile " SOURCES STR " . $sources . "\n"; + if ($needConvolvedImages) { + print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; + print $listFile " CNV:MASK STR " . $maskCnv . "\n"; + print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; + print $listFile " CNV:EXPNUM STR " . $expnumCnv . "\n"; + print $listFile " CNV:PSF STR " . $psfCnv . "\n"; + } print $listFile "END\n\n"; Index: branches/eam_branches/ipp-20131211/ippTasks/Makefile.am =================================================================== --- branches/eam_branches/ipp-20131211/ippTasks/Makefile.am (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTasks/Makefile.am (revision 36480) @@ -46,6 +46,8 @@ diffphot.pro \ lap.pro \ + lapgroup.pro \ vp.pro \ - bg.regeneration.pro + bg.regeneration.pro \ + fullforce.pro other_files = \ Index: branches/eam_branches/ipp-20131211/ippTasks/fullforce.pro =================================================================== --- branches/eam_branches/ipp-20131211/ippTasks/fullforce.pro (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTasks/fullforce.pro (revision 36480) @@ -0,0 +1,458 @@ +## fullforce.pro : tasks for static sky calibration analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the fullforce analysis. + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init fullForceRun +book init fullForceSummary + +### Database lists +$fullForce_DB = 0 +$fullForce_revert_DB = 0 +$fullForceSummary_DB = 0 +$fullForceSummary_revert_DB = 0 + +### Check status of fullForce tasks +macro fullforce.status + book listbook fullForceRun +end + +### Reset fullForce tasks +macro fullforce.reset + book init fullForceRun +end +macro fullforce.summary.status + book listbook fullForceSummary +end + +### Reset fullForce tasks +macro fullforce.summary.reset + book init fullForceSummary +end + +### Turn fullForce tasks on +macro fullforce.on + task fullforce.load + active true + end + task fullforce.run + active true + end + task fullforce.summary.load + active true + end + task fullforce.summary.run + active true + end +end + +### Turn fullForce tasks off +macro fullforce.off + task fullforce.load + active false + end + task fullforce.run + active false + end + task fullforce.summary.load + active false + end + task fullforce.summary.run + active false + end +end + +macro fullforce.revert.on + task fullforce.revert + active true + end + task fullforce.summary.revert + active true + end +end + +macro fullforce.revert.off + task fullforce.revert + active false + end + task fullforce.summary.revert + active false + end +end + +### Load tasks for fullforce +### Tasks are loaded into fullForceRun. +task fullforce.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fullforce.log + + task.exec + if ($LABEL:n == 0) break + $run = fftool -todo + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForce_DB + $run = $run -dbname $DB:$fullForce_DB + $fullForce_DB ++ + if ($fullForce_DB >= $DB:n) set fullForce_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fullForceRun -key ff_id:warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fullForceRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fullForceRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the fullforce analysis +### Tasks are taken from fullForceRun. +task fullforce.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages fullForceRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in fullForceRun (pantaskState == INIT) + book getpage fullForceRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fullForceRun $pageName pantaskState RUN + book getword fullForceRun $pageName ff_id -var FF_ID + book getword fullForceRun $pageName warp_id -var WARP_ID + book getword fullForceRun $pageName sources_path_base -var SOURCES_PATH_BASE + book getword fullForceRun $pageName tess_id -var TESS_DIR + book getword fullForceRun $pageName skycell_id -var SKYCELL_ID + book getword fullForceRun $pageName camera -var CAMERA + book getword fullForceRun $pageName workdir -var WORKDIR_TEMPLATE + book getword fullForceRun $pageName warp_path_base -var WARP_PATH_BASE + book getword fullForceRun $pageName reduction -var REDUCTION + book getword fullForceRun $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.wrp.%s.ff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID $FF_ID + + stdout $LOGDIR/fullforce.log + stderr $LOGDIR/fullforce.log + + $run = psphot_fullforce_warp.pl --ff_id $FF_ID --warp_id $WARP_ID --outroot $outroot --redirect-output --camera $CAMERA --warp_path_base $WARP_PATH_BASE --sources_path_base $SOURCES_PATH_BASE --skycell_id $SKYCELL_ID --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec .05 + command $run + end + + # default exit status + task.exit default + process_exit fullForceRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword fullForceRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fullForceRun $options:0 pantaskState TIMEOUT + end +end + +task fullforce.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/fullforce.revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = fftool -revert -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForce_revert_DB + $run = $run -dbname $DB:$fullForce_revert_DB + $fullForce_revert_DB ++ + if ($fullForce_revert_DB >= $DB:n) set fullForce_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Load tasks for fullForceSummary +### Tasks are loaded into fullForceSummary. +task fullforce.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fullforce.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = fftool -toadvance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForceSummary_DB + $run = $run -dbname $DB:$fullForceSummary_DB + $fullForceSummary_DB ++ + if ($fullForceSummary_DB >= $DB:n) set fullForceSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fullForceSummary -key ff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fullForceSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fullForceSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the fullForceSummary analyasis +### Tasks are taken from fullForceSummary. +task fullforce.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages fullForceSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in fullForceSummary (pantaskState == INIT) + book getpage fullForceSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fullForceSummary $pageName pantaskState RUN + book getword fullForceSummary $pageName ff_id -var FF_ID + book getword fullForceSummary $pageName tess_id -var TESS_DIR + book getword fullForceSummary $pageName skycell_id -var SKYCELL_ID + book getword fullForceSummary $pageName camera -var CAMERA + book getword fullForceSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword fullForceSummary $pageName reduction -var REDUCTION + book getword fullForceSummary $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.ff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $FF_ID + + stdout $LOGDIR/fullforce.summary.log + stderr $LOGDIR/fullforce.summary.log + + $run = psphot_fullforce_summary.pl --ff_id $FF_ID --outroot $outroot --redirect-output --camera $CAMERA --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec .05 + command $run + end + + # default exit status + task.exit default + process_exit fullForceSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword fullForceSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fullForceSummary $options:0 pantaskState TIMEOUT + end +end + +task fullforce.summary.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/fullforce.summary.revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = fftool -revertsummary -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForceSummary_revert_DB + $run = $run -dbname $DB:$fullForceSummary_revert_DB + $fullForceSummary_revert_DB ++ + if ($fullForceSummary_revert_DB >= $DB:n) set fullForceSummary_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: branches/eam_branches/ipp-20131211/ippTasks/pstamp.pro =================================================================== --- branches/eam_branches/ipp-20131211/ippTasks/pstamp.pro (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTasks/pstamp.pro (revision 36480) @@ -825,4 +825,5 @@ book getword pstampDependent $pageName imagedb -var IMAGEDB book getword pstampDependent $pageName rlabel -var RLABEL + book getword pstampDependent $pageName label -var LABEL book getword pstampDependent $pageName outdir -var OUTDIR book getword pstampDependent $pageName need_magic -var NEED_MAGIC @@ -846,5 +847,5 @@ stderr $MYLOGFILE - $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT --logfile $MYLOGFILE + $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL --label $LABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT --logfile $MYLOGFILE add_standard_args run Index: branches/eam_branches/ipp-20131211/ippTasks/rawcheck.pro =================================================================== --- branches/eam_branches/ipp-20131211/ippTasks/rawcheck.pro (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTasks/rawcheck.pro (revision 36480) @@ -75,5 +75,5 @@ task.exec book npages rawcheckPending -var N - if ($N > 2000) + if ($N > 200) process_cleanup rawcheckPending break Index: branches/eam_branches/ipp-20131211/ippToPsps/jython/dvo.py =================================================================== --- branches/eam_branches/ipp-20131211/ippToPsps/jython/dvo.py (revision 36479) +++ branches/eam_branches/ipp-20131211/ippToPsps/jython/dvo.py (revision 36480) @@ -65,5 +65,5 @@ if not self.correctDvo: print "*******************************************************************************" - self.logger.warning("switching to a new dvo: " + self.skychunk.dvoLocation) + self.logger.warning("switching to a new dvo: '" + self.skychunk.dvoLocation + "'") # response = raw_input("(y/n) ") # if response == "y": @@ -669,5 +669,24 @@ cmd += " -time-start " + time_start cmd += " -time-end " + time_end - + + useP2 = 0 + useST = 0 + + for batchType in self.skychunk.batchTypes: + self.logger.infoPair("batchType", batchType) + if batchType == "ST": + useST = 1 + if batchType == "P2": + useP2 = 1 + + if (useP2 ==0 and useST ==1): + #grab only stacks + cmd += " -photcode-start 11000 -photcode-end 11500" + + if (useP2 ==1 and useST==0): + #grabd only P2s + cmd += " -photcode-start 10000 -photcode-end 10577" + + self.logger.infoPair("Running dvopsps", cmd) p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) Index: branches/eam_branches/ipp-20131211/ippToPsps/jython/gpc1db.py =================================================================== --- branches/eam_branches/ipp-20131211/ippToPsps/jython/gpc1db.py (revision 36479) +++ branches/eam_branches/ipp-20131211/ippToPsps/jython/gpc1db.py (revision 36480) @@ -93,5 +93,7 @@ JOIN mergedvodbRun USING(minidvodb_id) \ JOIN mergedvodbProcessed USING (merge_id) \ - JOIN stackRun USING(stack_id) JOIN skycell USING(skycell_id) \ + JOIN stackRun USING(stack_id) \ + JOIN stackSumSkyfile using (stack_id) \ + JOIN skycell USING(skycell_id) \ WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ AND minidvodbRun.state = 'merged' \ @@ -102,6 +104,8 @@ AND addRun.state = 'full' \ AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ - AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) - + AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + AND mjd_obs >= (to_days('" + tstart + "')-678941) \ + AND mjd_obs <= (to_days('" + tend + "') - 678941) " + self.logger.infoPair("sql",sql) try: # XXX EAM : test output Index: branches/eam_branches/ipp-20131211/ippToPsps/jython/queue.py =================================================================== --- branches/eam_branches/ipp-20131211/ippToPsps/jython/queue.py (revision 36479) +++ branches/eam_branches/ipp-20131211/ippToPsps/jython/queue.py (revision 36480) @@ -45,4 +45,5 @@ try: + print "why doesn't this work" self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) except: Index: branches/eam_branches/ipp-20131211/ippToPsps/jython/stackbatch.py =================================================================== --- branches/eam_branches/ipp-20131211/ippToPsps/jython/stackbatch.py (revision 36479) +++ branches/eam_branches/ipp-20131211/ippToPsps/jython/stackbatch.py (revision 36480) @@ -303,4 +303,50 @@ modelLong = model; + #does it have the EXT_COVAR in there? + + sql = "DESCRIBE SkyChip_xfit 'EXT_COVAR_%'" + rs = self.scratchDb.executeQuery(sql) + rs.first() + hasResult = rs.next() + if hasResult: + print "EXT_COVARs exist" + else: + print "No EXT_COVARS here" + + ext_covar_sql = "" + + if (hasResult): + ext_covar_sql = ", "+modelLong+"Covar11=b.EXT_COVAR_00_00, \ + "+modelLong+"Covar12=b.EXT_COVAR_00_01, \ + "+modelLong+"Covar13=b.EXT_COVAR_00_02, \ + "+modelLong+"Covar14=b.EXT_COVAR_00_03, \ + "+modelLong+"Covar15=b.EXT_COVAR_00_04, \ + "+modelLong+"Covar16=b.EXT_COVAR_00_05, \ + "+modelLong+"Covar17=b.EXT_COVAR_00_06, \ + "+modelLong+"Covar22=b.EXT_COVAR_01_01, \ + "+modelLong+"Covar23=b.EXT_COVAR_01_02, \ + "+modelLong+"Covar24=b.EXT_COVAR_01_03, \ + "+modelLong+"Covar25=b.EXT_COVAR_01_04, \ + "+modelLong+"Covar26=b.EXT_COVAR_01_05, \ + "+modelLong+"Covar27=b.EXT_COVAR_01_06, \ + "+modelLong+"Covar33=b.EXT_COVAR_02_02, \ + "+modelLong+"Covar34=b.EXT_COVAR_02_03, \ + "+modelLong+"Covar35=b.EXT_COVAR_02_04, \ + "+modelLong+"Covar36=b.EXT_COVAR_02_05, \ + "+modelLong+"Covar37=b.EXT_COVAR_02_06, \ + "+modelLong+"Covar44=b.EXT_COVAR_03_03, \ + "+modelLong+"Covar45=b.EXT_COVAR_03_04, \ + "+modelLong+"Covar46=b.EXT_COVAR_03_05, \ + "+modelLong+"Covar47=b.EXT_COVAR_03_06, \ + "+modelLong+"Covar55=b.EXT_COVAR_04_04, \ + "+modelLong+"Covar56=b.EXT_COVAR_04_05, \ + "+modelLong+"Covar57=b.EXT_COVAR_04_06, \ + "+modelLong+"Covar66=b.EXT_COVAR_05_05, \ + "+modelLong+"Covar67=b.EXT_COVAR_05_06, \ + "+modelLong+"Covar77=b.EXT_COVAR_06_06" + + + + sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ "+model+"Radius=b.EXT_WIDTH_MAJ, \ @@ -308,39 +354,13 @@ "+model+"FluxErr=ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " / 1.085736, \ "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \ - "+model+"Phi=b.EXT_THETA, \ - "+modelLong+"Covar11=b.EXT_COVAR_00_00, \ - "+modelLong+"Covar12=b.EXT_COVAR_00_01, \ - "+modelLong+"Covar13=b.EXT_COVAR_00_02, \ - "+modelLong+"Covar14=b.EXT_COVAR_00_03, \ - "+modelLong+"Covar15=b.EXT_COVAR_00_04, \ - "+modelLong+"Covar16=b.EXT_COVAR_00_05, \ - "+modelLong+"Covar17=b.EXT_COVAR_00_06, \ - "+modelLong+"Covar22=b.EXT_COVAR_01_01, \ - "+modelLong+"Covar23=b.EXT_COVAR_01_02, \ - "+modelLong+"Covar24=b.EXT_COVAR_01_03, \ - "+modelLong+"Covar25=b.EXT_COVAR_01_04, \ - "+modelLong+"Covar26=b.EXT_COVAR_01_05, \ - "+modelLong+"Covar27=b.EXT_COVAR_01_06, \ - "+modelLong+"Covar33=b.EXT_COVAR_02_02, \ - "+modelLong+"Covar34=b.EXT_COVAR_02_03, \ - "+modelLong+"Covar35=b.EXT_COVAR_02_04, \ - "+modelLong+"Covar36=b.EXT_COVAR_02_05, \ - "+modelLong+"Covar37=b.EXT_COVAR_02_06, \ - "+modelLong+"Covar44=b.EXT_COVAR_03_03, \ - "+modelLong+"Covar45=b.EXT_COVAR_03_04, \ - "+modelLong+"Covar46=b.EXT_COVAR_03_05, \ - "+modelLong+"Covar47=b.EXT_COVAR_03_06, \ - "+modelLong+"Covar55=b.EXT_COVAR_04_04, \ - "+modelLong+"Covar56=b.EXT_COVAR_04_05, \ - "+modelLong+"Covar57=b.EXT_COVAR_04_06, \ - "+modelLong+"Covar66=b.EXT_COVAR_05_05, \ - "+modelLong+"Covar67=b.EXT_COVAR_05_06, \ - "+modelLong+"Covar77=b.EXT_COVAR_06_06 \ + "+model+"Phi=b.EXT_THETA \ + "+ext_covar_sql+" \ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" + print sql self.scratchDb.execute(sql) # sersic fit has an extra parameter - if ippModelType == "PS_MODEL_SERSIC": + if (ippModelType == "PS_MODEL_SERSIC" and hasResult > 0): sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ "+modelLong+"Covar18=b.EXT_COVAR_00_07, \ Index: branches/eam_branches/ipp-20131211/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/Makefile.am (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTools/share/Makefile.am (revision 36480) @@ -488,4 +488,12 @@ releasetool_definerelgroup_select_exp_lap.sql \ releasetool_stacksummary.sql \ - releasetool_pendingrelgroup.sql + releasetool_pendingrelgroup.sql \ + fftool_definebyquery.sql \ + fftool_definebyquery_select_warps.sql \ + fftool_result.sql \ + fftool_revert.sql \ + fftool_revertsummary.sql \ + fftool_summary.sql \ + fftool_todo.sql \ + fftool_toadvance.sql Index: branches/eam_branches/ipp-20131211/ippTools/share/chiptool_pendingcleanuprun.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/chiptool_pendingcleanuprun.sql (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTools/share/chiptool_pendingcleanuprun.sql (revision 36480) @@ -4,8 +4,11 @@ rawExp.exp_tag, chipRun.state, - chipRun.workdir + chipRun.workdir, + chipRun.label, + IFNULL(Label.priority, 10000) AS priority FROM chipRun JOIN rawExp USING (exp_id) +LEFT JOIN Label ON chipRun.label = Label.label WHERE (chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_scrubbed' OR chipRun.state = 'goto_purged') Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_definebyquery.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_definebyquery.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_definebyquery.sql (revision 36480) @@ -0,0 +1,14 @@ +SELECT + skycalRun.skycal_id, + skycalResult.path_base, + skycalRun.data_group, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter +FROM skycalRun + JOIN skycalResult USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) + -- join hook %s +WHERE + skycalRun.state = 'full' AND skycalResult.quality = 0 Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_definebyquery_select_warps.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_definebyquery_select_warps.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_definebyquery_select_warps.sql (revision 36480) @@ -0,0 +1,8 @@ +SELECT warp_id +FROM warpRun join warpSkyfile using(warp_id, tess_id) + JOIN fakeRun using(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE warpRun.tess_id = '%s' AND warpSkyfile.skycell_id = '%s' AND rawExp.filter = '%s' + AND warpRun.state = 'full' AND warpSkyfile.quality = 0 and warpSkyfile.fault = 0 Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_result.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_result.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_result.sql (revision 36480) @@ -0,0 +1,19 @@ +SELECT + fullForceResult.*, + fullForceRun.skycal_id, + fullForceRun.label, + fullForceRun.data_group, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceResult USING(ff_id) + JOIN skycalRun using(skycal_id) + JOIN stackRun using(stack_id) + JOIN warpRun using(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun using(fake_id) + JOIN camRun using(cam_id) + JOIN chipRun using(chip_id) + JOIN rawExp using(exp_id) Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_revert.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_revert.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_revert.sql (revision 36480) @@ -0,0 +1,7 @@ +DELETE FROM fullForceResult +USING fullForceResult, fullForceRun, skycalRun, stackRun, skycell +WHERE fullForceRun.ff_id = fullForceResult.ff_id + AND fullForceResult.fault != 0 + AND fullForceRun.skycal_id = skycalRun.skycal_id + AND skycalRun.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_revertsummary.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_revertsummary.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_revertsummary.sql (revision 36480) @@ -0,0 +1,7 @@ +DELETE FROM fullForceSummary +USING fullForceSummary, fullForceRun, skycalRun, stackRun, skycell +WHERE fullForceRun.ff_id = fullForceSummary.ff_id + AND fullForceSummary.fault != 0 + AND fullForceRun.skycal_id = skycalRun.skycal_id + AND skycalRun.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_summary.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_summary.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_summary.sql (revision 36480) @@ -0,0 +1,21 @@ +SELECT DISTINCT + fullForceSummary.*, + fullForceRun.state, + fullForceRun.label, + fullForceRun.data_group, + fullForceRun.skycal_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceSummary USING(ff_id) + JOIN fullForceInput USING(ff_id) + JOIN skycalRun using(skycal_id) + JOIN stackRun using(stack_id) + JOIN warpRun using(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun using(fake_id) + JOIN camRun using(cam_id) + JOIN chipRun using(chip_id) + JOIN rawExp using(exp_id) Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_summaryinputs.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_summaryinputs.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_summaryinputs.sql (revision 36480) @@ -0,0 +1,4 @@ +SELECT + fullForceResult.* +FROM fullForceRun + JOIN fullForceResult USING(ff_id) Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_toadvance.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_toadvance.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_toadvance.sql (revision 36480) @@ -0,0 +1,21 @@ +SELECT + fullForceRun.*, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceInput USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN warpRun USING(warp_id, tess_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN fullForceResult USING(ff_id, warp_id) +WHERE fullForceRun.state = 'new' + -- WHERE hook %s + GROUP BY ff_id + HAVING COUNT(fullForceInput.warp_id) = COUNT(fullForceResult.warp_id) + AND SUM(fullForceResult.fault) = 0 Index: branches/eam_branches/ipp-20131211/ippTools/share/fftool_todo.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/fftool_todo.sql (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/share/fftool_todo.sql (revision 36480) @@ -0,0 +1,22 @@ +SELECT + fullForceRun.*, + fullForceInput.warp_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera, + warpSkyfile.path_base AS warp_path_base +FROM fullForceRun + JOIN fullForceInput USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN warpRun USING(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN fullForceResult USING(ff_id, warp_id) +WHERE fullForceRun.state = 'new' + AND fullForceResult.ff_id IS NULL + AND warpSkyfile.data_state = 'full' Index: branches/eam_branches/ipp-20131211/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/pxadmin_create_tables.sql (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTools/share/pxadmin_create_tables.sql (revision 36480) @@ -2307,4 +2307,62 @@ ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +CREATE TABLE fullForceRun ( + ff_id BIGINT NOT NULL AUTO_INCREMENT, + skycal_id BIGINT, + sources_path_base VARCHAR(255), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + note VARCHAR(255), + reduction VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(ff_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(skycal_id), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceInput ( + ff_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(ff_id, warp_id), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceResult ( + ff_id BIGINT, + warp_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id, warp_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceSummary ( + ff_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + -- These comment lines are here to avoid an empty query error. Index: branches/eam_branches/ipp-20131211/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/share/pxadmin_drop_tables.sql (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTools/share/pxadmin_drop_tables.sql (revision 36480) @@ -120,4 +120,9 @@ DROP TABLE IF EXISTS dqstatsContent; DROP TABLE IF EXISTS dqstatsRun; +DROP TABLE IF EXISTS fullForceRun; +DROP TABLE IF EXISTS fullForceInput; +DROP TABLE IF EXISTS fullForceResult; +DROP TABLE IF EXISTS fullForceSummary; + SET FOREIGN_KEY_CHECKS=1 Index: branches/eam_branches/ipp-20131211/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/src/Makefile.am (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTools/src/Makefile.am (revision 36480) @@ -33,5 +33,6 @@ vptool \ sctool \ - releasetool + releasetool \ + fftool pkginclude_HEADERS = \ @@ -87,5 +88,6 @@ vptool.h \ sctool.h \ - releasetool.h + releasetool.h \ + fftool.h lib_LTLIBRARIES = libpxtools.la @@ -332,4 +334,10 @@ releasetoolConfig.c +fftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +fftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +fftool_SOURCES = \ + fftool.c \ + fftoolConfig.c + clean-local: -rm -f TAGS Index: branches/eam_branches/ipp-20131211/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/src/chiptool.c (revision 36479) +++ branches/eam_branches/ipp-20131211/ippTools/src/chiptool.c (revision 36480) @@ -1255,5 +1255,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); psString query = pxDataGet("chiptool_pendingcleanuprun.sql"); @@ -1269,4 +1269,6 @@ } psFree(where); + + psStringAppend(&query, "\nORDER BY priority DESC, chip_id"); // treat limit == 0 as "no limit" Index: branches/eam_branches/ipp-20131211/ippTools/src/fftool.c =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/src/fftool.c (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/src/fftool.c (revision 36480) @@ -0,0 +1,934 @@ +/* + * fftool.c + * + * Copyright (C) 2013 Institute for Astronomy, University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxspace.h" +#include "fftool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool todoMode(pxConfig *config); +static bool addresultMode(pxConfig *config); +static bool resultMode(pxConfig *config); +static bool revertMode(pxConfig *config); +static bool updateresultMode(pxConfig *config); +static bool toadvanceMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); +static bool revertsummaryMode(pxConfig *config); +static bool updatesummaryMode(pxConfig *config); +static bool summaryMode(pxConfig *config); + +static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = fftoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(FFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(FFTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(FFTOOL_MODE_TODO, todoMode); + MODECASE(FFTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(FFTOOL_MODE_RESULT, resultMode); + MODECASE(FFTOOL_MODE_REVERT, revertMode); + MODECASE(FFTOOL_MODE_UPDATERESULT, updateresultMode); + MODECASE(FFTOOL_MODE_TOADVANCE, toadvanceMode); + MODECASE(FFTOOL_MODE_ADDSUMMARY, addsummaryMode); + MODECASE(FFTOOL_MODE_REVERTSUMMARY, revertsummaryMode); + MODECASE(FFTOOL_MODE_UPDATESUMMARY, updatesummaryMode); + MODECASE(FFTOOL_MODE_SUMMARY, summaryMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + // optional + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + + PXOPT_LOOKUP_STR(sources_path_base, config->args, "-set_sources_path_base", false, false); + + psMetadata *skycalWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_label", "skycalRun.label", "LIKE"); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, skycalWhereMD, "-select_skycal_id", "skycalRun.skycal_id", "=="); + + if (!psListLength(skycalWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required"); + psFree(skycalWhereMD); + return false; + } + + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_filter", "stackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, skycalWhereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + psFree(skycalWhereMD); + return false; + } + + psMetadata *warpWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","warpRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "warpRun.warp_id", "=="); + if (!psListLength(warpWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required"); + psFree(warpWhereMD); + psFree(skycalWhereMD); + return false; + } + if (!psListLength(warpWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required"); + psFree(skycalWhereMD); + psFree(warpWhereMD); + return false; + } + + PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "warpRun.tess_id", "=="); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "warpRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, warpWhereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + psFree(skycalWhereMD); + psFree(warpWhereMD); + return false; + } + + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString select = pxDataGet("fftool_definebyquery.sql"); + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(skycalWhereMD); + psFree(warpWhereMD); + return false; + } + + psString where = NULL; + psString whereClause = psDBGenerateWhereConditionSQL(skycalWhereMD, NULL); + psStringAppend(&where, "\nAND %s", whereClause); + psStringAppend(&select, where); + psFree(whereClause); + psFree(skycalWhereMD); + + psString joinHook = NULL; + if (!rerun) { + psStringAppend(&joinHook, "\nLEFT JOIN fullForceRun ON fullForceRun.skycal_id = fullForceRun.skycal_id"); + psStringAppend(&joinHook, "\n %s\nAND fullForceRun.label = '%s'", where, label); + psStringAppend(&select, "\nAND ff_id IS NULL"); + } + + if (!p_psDBRunQueryF(config->dbh, select, joinHook)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + return false; + } + psFree(select); + psFree(joinHook); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("fftool: no rows found"); + psFree(output); + psFree(where); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "toFullForce", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } + + psString warpQueryTemplate = pxDataGet("fftool_definebyquery_select_warps.sql"); + + whereClause = psDBGenerateWhereConditionSQL(warpWhereMD, NULL); + psStringAppend(&warpQueryTemplate, "\nAND %s", whereClause); + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); + psS64 skycal_id = psMetadataLookupS64(&status, row, "skycal_id"); + + psString path_base = NULL; + if (sources_path_base) { + path_base = sources_path_base; + } else { + path_base = psMetadataLookupStr(&status, row, "path_base"); + psAssert(status, "failed to find skycal path_base?"); + } + + psString skycal_data_group = psMetadataLookupStr(&status, row, "data_group"); + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + psString filter = psMetadataLookupStr(&status, row, "filter"); + + psString query = NULL; + psStringAppend(&query, warpQueryTemplate, tess_id, skycell_id, filter); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // Find the warps for this skycell and filter combination + psArray *warpOutput = p_psDBFetchResult(config->dbh); + if (!warpOutput) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(warpOutput)) { + // no warps for this skycal. Suprise? + psFree(warpOutput); + psFree(query); + continue; + } + + // create a staticskyRun + if (!fullForceRunInsert(config->dbh, + 0x0, // ff_id + skycal_id, + path_base, + "new", // state + workdir, + label, + data_group ? data_group : (skycal_data_group ? skycal_data_group : label), + dist_group, + note, + reduction, + NULL // registered + ) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + psS64 ff_id = psDBLastInsertID(config->dbh); + + for (int j = 0; j < warpOutput->n; j++) { + psMetadata *warpRow = warpOutput->data[j]; + psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id"); + + if (!fullForceInputInsert(config->dbh, + ff_id, + warp_id) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + psFree(warpOutput); + } + psFree(output); + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "fullForceRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE fullForceRun JOIN skycalRun USING(skycal_id) JOIN stackRun USING(stack_id) JOIN skycell USING(tess_id, skycell_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "fullForceRun", "ff_id", "fullForceResult", true, false); + psFree(query); + psFree(where); + + return result; +} + + +static bool todoMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "fullForceRun.label", "=="); + pxskycellAddWhere(config, whereMD); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_todo.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fullForceRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // required + PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", true, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(software_ver, config->args, "-software_ver", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + // XXX not sure we need a transaction here... + if (!fullForceResultInsert(config->dbh, + ff_id, + warp_id, + path_base, + dtime_script, + quality, + hostname, + software_ver, + fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool resultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "fullForceRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + pxskycellAddWhere(config, where); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_result.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "fullForceResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool revertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceResult.warp_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceResult.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("fftool_revert.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("fftool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} + +static bool updateresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + + if (!pxSetFaultCode(config->dbh, "fullForceResult", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} + +static bool setfullForceRunState(pxConfig *config, psS64 ff_id, const char *state) +{ + psString query = "UPDATE fullForceRun SET state = 'full' WHERE ff_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, ff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool toadvanceMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "fullForceRun.label", "=="); + pxskycellAddWhere(config, whereMD); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_toadvance.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = NULL; + psString temp = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&whereClause, "\n AND %s", temp); + psFree(temp); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + // the where clause is required and is added to the query by the "WHERE hook" format string + if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fullForceRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + // required + PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", true, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(software_ver, config->args, "-software_ver", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fullForceSummaryInsert(config->dbh, + ff_id, + path_base, + dtime_script, + quality, + hostname, + software_ver, + fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fault) { + if (!setfullForceRunState(config, ff_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change staticskyRun state"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool revertsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceSummary.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("fftool_revertsummary.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("fftool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} +static bool updatesummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + + + if (!pxSetFaultCode(config->dbh, "fullForceSummary", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} +static bool summaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceInput.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "fullForceRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + pxskycellAddWhere(config, where); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_summary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "fullForceSummary", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: branches/eam_branches/ipp-20131211/ippTools/src/fftool.h =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/src/fftool.h (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/src/fftool.h (revision 36480) @@ -0,0 +1,43 @@ +/* + * fftool.h + * + * Copyright (C) 2013 IfA Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FFTOOL_H +#define FFTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + FFTOOL_MODE_NONE = 0x0, + FFTOOL_MODE_DEFINEBYQUERY, + FFTOOL_MODE_UPDATERUN, + FFTOOL_MODE_TODO, + FFTOOL_MODE_ADDRESULT, + FFTOOL_MODE_RESULT, + FFTOOL_MODE_REVERT, + FFTOOL_MODE_UPDATERESULT, + FFTOOL_MODE_TOADVANCE, + FFTOOL_MODE_ADDSUMMARY, + FFTOOL_MODE_REVERTSUMMARY, + FFTOOL_MODE_UPDATESUMMARY, + FFTOOL_MODE_SUMMARY +} fftoolMode; + +pxConfig *fftoolConfig(pxConfig *config, int argc, char **argv); + +#endif // FFTOOL_H Index: branches/eam_branches/ipp-20131211/ippTools/src/fftoolConfig.c =================================================================== --- branches/eam_branches/ipp-20131211/ippTools/src/fftoolConfig.c (revision 36480) +++ branches/eam_branches/ipp-20131211/ippTools/src/fftoolConfig.c (revision 36480) @@ -0,0 +1,228 @@ +/* + * fftool.c + * + * Copyright (C) 2013 Institute for Astronomy, University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "fftool.h" + +pxConfig *fftoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + (void) now; + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_label", PS_META_DUPLICATE_OK, "search by warp label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_label", PS_META_DUPLICATE_OK, "search by skycal label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_data_group", PS_META_DUPLICATE_OK, "search by warp data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_data_group", PS_META_DUPLICATE_OK, "search by skycal data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_id", 0, "search by skycal ID", 0); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + + pxskycellAddArguments(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_sources_path_base", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + pxskycellAddArguments(updaterunArgs); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + + // -todo + psMetadata *todoArgs = psMetadataAlloc(); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(todoArgs); + psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(todoArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresult + psMetadata *addresultArgs = psMetadataAlloc(); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -result + psMetadata *resultArgs= psMetadataAlloc(); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + pxskycellAddArguments(resultArgs); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revert + psMetadata *revertArgs= psMetadataAlloc(); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertArgs); + + // -updateresult + psMetadata *updateresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality code", 0); + + // -toadvance + psMetadata *toadvanceArgs = psMetadataAlloc(); + psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(toadvanceArgs); + psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); + psMetadataAddF32(addsummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -updatesummary + psMetadata *updatesummaryArgs = psMetadataAlloc(); + psMetadataAddS64(updatesummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF32(updatesummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -summary + psMetadata *summaryArgs= psMetadataAlloc(); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + pxskycellAddArguments(summaryArgs); + psMetadataAddS16(summaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertsummary + psMetadata *revertsummaryArgs= psMetadataAlloc(); + psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(revertsummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertsummaryArgs); + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "Define a new full force run", FFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "Update a run", FFTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-todo", "Get runs to do", FFTOOL_MODE_TODO, todoArgs); + PXOPT_ADD_MODE("-addresult", "Add result for fullforce run on a warp", FFTOOL_MODE_ADDRESULT, addresultArgs); + PXOPT_ADD_MODE("-result", "Get result fullforce run on a warp", FFTOOL_MODE_RESULT, resultArgs); + PXOPT_ADD_MODE("-revert", "Revert failed fullforce run on a warp", FFTOOL_MODE_REVERT, revertArgs); + PXOPT_ADD_MODE("-updateresult", "Update result for fullforce run on a warp", FFTOOL_MODE_UPDATERESULT, updateresultArgs); + PXOPT_ADD_MODE("-toadvance", "list completed runs to summarize", FFTOOL_MODE_TOADVANCE, toadvanceArgs); + PXOPT_ADD_MODE("-addsummary", "insert summary result", FFTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-updatesummary", "update summary result", FFTOOL_MODE_UPDATESUMMARY, updatesummaryArgs); + PXOPT_ADD_MODE("-revertsummary", "revert faulted summary", FFTOOL_MODE_REVERTSUMMARY, revertsummaryArgs); + PXOPT_ADD_MODE("-summary", "list summary results", FFTOOL_MODE_SUMMARY, summaryArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/ipp-20131211/ippconfig/gpc1/psastro.config =================================================================== --- branches/eam_branches/ipp-20131211/ippconfig/gpc1/psastro.config (revision 36479) +++ branches/eam_branches/ipp-20131211/ippconfig/gpc1/psastro.config (revision 36480) @@ -380,4 +380,13 @@ END +POLE_REFCAT METADATA + PSASTRO.CATDIR STR SYNTH.GRIZY + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.GRID.MIN.ANGLE F32 -20 + PSASTRO.GRID.MAX.ANGLE F32 +20 + PSASTRO.GRID.DEL.ANGLE F32 0.5 + PSASTRO.FIELD.PADDING F32 0.5 +END + TEST_REFCAT METADATA PSASTRO.CATDIR STR /data/ipp064.0/ipp/ippRefs/catdir.refcat.20120524.v0 Index: branches/eam_branches/ipp-20131211/ippconfig/recipes/filerules-split.mdc =================================================================== --- branches/eam_branches/ipp-20131211/ippconfig/recipes/filerules-split.mdc (revision 36479) +++ branches/eam_branches/ipp-20131211/ippconfig/recipes/filerules-split.mdc (revision 36480) @@ -262,4 +262,6 @@ PSPHOT.STACK.RESID OUTPUT {OUTPUT}.stk.{FILE.ID}.res.fits IMAGE COMP_SUB FPA TRUE NONE PSPHOT.STACK.CONFIG OUTPUT {OUTPUT}.psphotStack.mdc TEXT NONE FPA TRUE NONE + +PSPHOT.FULLFORCE.OUTPUT OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE SOURCE.PLOT.RAW.MOMENTS OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png KAPA NONE CHIP TRUE NONE Index: branches/eam_branches/ipp-20131211/ippconfig/recipes/psastro.config =================================================================== --- branches/eam_branches/ipp-20131211/ippconfig/recipes/psastro.config (revision 36479) +++ branches/eam_branches/ipp-20131211/ippconfig/recipes/psastro.config (revision 36480) @@ -3,4 +3,5 @@ PSASTRO.ONLY.REFSTARS BOOL FALSE # skip all but refstar matches PSASTRO.SAVE.REFMATCH BOOL FALSE # save refstar matches as table in output smf file +PSASTRO.SAVE.CFF BOOL FALSE # save a cff file (input for psphotFullForce) # select which WCS style to use on output images. @@ -272,8 +273,12 @@ END +POLE_REFCAT METADATA +END + TEST_REFCAT METADATA END STATICSKY_CAL METADATA + PSASTRO.SAVE.CFF BOOL TRUE # save a cff file (input for psphotFullForce) END Index: branches/eam_branches/ipp-20131211/ippconfig/recipes/psphot.config =================================================================== --- branches/eam_branches/ipp-20131211/ippconfig/recipes/psphot.config (revision 36479) +++ branches/eam_branches/ipp-20131211/ippconfig/recipes/psphot.config (revision 36480) @@ -196,6 +196,15 @@ EXT.NSIGMA.LIMIT.USE BOOL TRUE -EXT.FIT.MIN.GAL.LIMIT F32 10.0 -EXT.FIT.MIN.GAL.LIMIT.USE BOOL FALSE +# Limits on extended source fits by galactic coordinates +EXT.FIT.MIN.GAL.LIMIT.USE BOOL FALSE # if true apply the galactic latitude cut +# objects are skipped for extended source fits if they have a abs(galactic latitude) larger than +# a limit value which is a function of galactic longitude. +# The galactic latitude limit is the sum of the following constant value ... +EXT.FIT.MIN.GAL.LIMIT F32 20.0 +# ... and a gaussian function of galactic longitude centered at b = 0 with this magnitude +EXT.FIT.MIN.GAL.LIMIT.BULGE F32 15.0 +# ... and this sigma +EXT.FIT.MIN.GAL.LIMIT.BULGE.SIGMA F32 50.0 # with this sigma value + KRON_ITERATIONS S32 2 @@ -224,7 +233,7 @@ #QGAUSS_PSF EXTENDED_SOURCE_MODEL PS_MODEL_QGAUSS 20.0 TRUE # TRAIL_RAW EXTENDED_SOURCE_MODEL PS_MODEL_TRAIL 10.0 FALSE - EXP_PCM EXTENDED_SOURCE_MODEL PS_MODEL_EXP 10.0 TRUE - DEV_PCM EXTENDED_SOURCE_MODEL PS_MODEL_DEV 10.0 TRUE - SER_PCM EXTENDED_SOURCE_MODEL PS_MODEL_SERSIC 10.0 TRUE + EXP_PCM EXTENDED_SOURCE_MODEL PS_MODEL_EXP 0.0 TRUE + DEV_PCM EXTENDED_SOURCE_MODEL PS_MODEL_DEV 0.0 TRUE + SER_PCM EXTENDED_SOURCE_MODEL PS_MODEL_SERSIC 0.0 TRUE END @@ -407,12 +416,12 @@ EXT.ANALYSIS.MAG.LIMITS METADATA - TYPE DATA FILTER.ID MAG.LIMIT - gband DATA g NAN - rband DATA r NAN - iband DATA i NAN - zband DATA z NAN - yband DATA y NAN - wband DATA w NAN - other DATA any NAN + TYPE DATA FILTER.ID MAG.LIMIT.PETRO MAG.LIMIT.EXTFIT + gband DATA g 25 21.5 + rband DATA r 25 21.5 + iband DATA i 25 21.5 + zband DATA z 25 20.5 + yband DATA y 25 19.5 + wband DATA w 25 21.5 + other DATA any 25 21.5 END @@ -423,12 +432,25 @@ RADIAL_APERTURES_SN_LIM F32 0.0 # S/N limit for radial aperture calculation +# zero point and exposure time to which fullForceSummary scales fluxes to +PSPHOT.FULLFORCE.EXPTIME F32 1.00 +PSPHOT.FULLFORCE.ZERO_PT F32 25.00 + + GALAXY_SHAPES BOOL F +GALAXY_SHAPES_STYLE STR R_MAJ_R_MIN + GALAXY_SHAPES_FR_MAJOR_MIN F32 0.5 -GALAXY_SHAPES_FR_MAJOR_MAX F32 2.0 -GALAXY_SHAPES_FR_MAJOR_DEL F32 0.1 +GALAXY_SHAPES_FR_MAJOR_MAX F32 2.00 +GALAXY_SHAPES_FR_MAJOR_DEL F32 0.05 GALAXY_SHAPES_FR_MINOR_MIN F32 0.5 -GALAXY_SHAPES_FR_MINOR_MAX F32 2.0 -GALAXY_SHAPES_FR_MINOR_DEL F32 0.1 - +GALAXY_SHAPES_FR_MINOR_MAX F32 2.00 +GALAXY_SHAPES_FR_MINOR_DEL F32 0.05 + +#GALAXY_SHAPES_FR_MAJOR_MIN F32 0.85 +#GALAXY_SHAPES_FR_MAJOR_MAX F32 1.15 +#GALAXY_SHAPES_FR_MAJOR_DEL F32 0.05 +#GALAXY_SHAPES_FR_MINOR_MIN F32 0.85 +#GALAXY_SHAPES_FR_MINOR_MAX F32 1.15 +#GALAXY_SHAPES_FR_MINOR_DEL F32 0.05 # Extended source fit parameters @@ -439,4 +461,5 @@ EXTENDED_SOURCE_ANNULI BOOL TRUE EXT.NSIGMA.LIMIT.USE BOOL FALSE + EXT.FIT.MIN.GAL.LIMIT.USE BOOL TRUE # limit extended source fits by galactic coordinates PSPHOT.STACK.MATCH.PSF.SOURCE STR AUTO # which inputs to convolve? (RAW, CNV, AUTO) @@ -465,4 +488,5 @@ EXT_MODEL STR PS_MODEL_QGAUSS PEAKS_NMAX_TOTAL S32 0 # set this to limit the allowed number of peaks - Yields fault instead of avoid memory explosion + SAVE.RESID BOOL TRUE @@ -683,8 +707,9 @@ RADIAL_APERTURES BOOL TRUE EXTENDED_SOURCE_FITS BOOL TRUE # this casues the xfit extension to be written out - EXTENDED_SOURCE_PETROSIAN BOOL TRUE # I want petrosian mags + EXTENDED_SOURCE_PETROSIAN BOOL TRUE # To measure petrosian magnitudes + EXTENDED_SOURCE_ANNULI BOOL TRUE SAVE.PSF BOOL FALSE SAVE.BACKMDL BOOL FALSE - # SAVE.RESID BOOL FALSE - # OUTPUT.FORMAT STR PS1_V3 -END + SAVE.RESID BOOL TRUE + OUTPUT.FORMAT STR PS1_SV2 +END Index: branches/eam_branches/ipp-20131211/ippconfig/recipes/reductionClasses.mdc =================================================================== --- branches/eam_branches/ipp-20131211/ippconfig/recipes/reductionClasses.mdc (revision 36479) +++ branches/eam_branches/ipp-20131211/ippconfig/recipes/reductionClasses.mdc (revision 36480) @@ -251,4 +251,14 @@ BACKGROUND_PSWARP STR BACKGROUND FULLFORCE_PSPHOT STR FULLFORCE_WARP +END + +# basic science analysis +POLE_REFCAT METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + ADDSTAR STR ADDSTAR + PSASTRO STR POLE_REFCAT END Index: branches/eam_branches/ipp-20131211/ppSub/src/ppSubMatchPSFs.c =================================================================== --- branches/eam_branches/ipp-20131211/ppSub/src/ppSubMatchPSFs.c (revision 36479) +++ branches/eam_branches/ipp-20131211/ppSub/src/ppSubMatchPSFs.c (revision 36480) @@ -156,6 +156,8 @@ psLogMsg("ppSub", PS_LOG_INFO, "Input FWHM: %f\nReference FWHM: %f\n", inFWHM, refFWHM); if (!isfinite(inFWHM) || !isfinite(refFWHM)) { - psError(PPSUB_ERR_DATA, false, "Cannot determine FHWM for images, giving up."); - return false; + psErrorStackPrint(stderr, "Cannot determine FHWM for images, giving up."); + int error = psErrorCodeLast(); // Error code + ppSubDataQuality(data, error, PPSUB_FILES_ALL); + return true; } Index: branches/eam_branches/ipp-20131211/psLib/src/math/psMixtureModels.c =================================================================== --- branches/eam_branches/ipp-20131211/psLib/src/math/psMixtureModels.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psLib/src/math/psMixtureModels.c (revision 36480) @@ -120,4 +120,5 @@ *Ncensored = 0; + int k = 0; for (int i = 0; i < in->numRows; i++) { int isCensored = 0; @@ -130,6 +131,11 @@ *Ncensored = *Ncensored + 1; } - offsets->data.S32[i] = *Ncensored; - } + else { + offsets->data.S32[k] = i - k; + k++; + } + // offsets->data.S32[i] = isCensored; + } + psImage *out; if (*Ncensored == 0) { @@ -320,6 +326,8 @@ } } - i = modes->data.F32[k]; - counts->data.F32[i] += 1.0; + if (modes->data.F32[k] != -1) { + i = modes->data.F32[k]; + counts->data.F32[i] += 1.0; + } } @@ -335,6 +343,8 @@ for (k = 0; k < N; k++) { i = modes->data.F32[k]; - for (j = 0; j < dim; j++) { - means->data.F32[i][j] += D->data.F32[k][j]; + if (i != -1) { + for (j = 0; j < dim; j++) { + means->data.F32[i][j] += D->data.F32[k][j]; + } } } Index: branches/eam_branches/ipp-20131211/psModules/src/camera/pmFPAfile.c =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/camera/pmFPAfile.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/camera/pmFPAfile.c (revision 36480) @@ -565,4 +565,6 @@ case PM_FPA_FILE_CMF: return ("CMF"); + case PM_FPA_FILE_CFF: + return ("CFF"); case PM_FPA_FILE_WCS: return ("WCS"); Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmModelClass.c =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmModelClass.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmModelClass.c (revision 36480) @@ -66,4 +66,5 @@ static pmModelClass *models = NULL; +static psVector *modelClassLookupTable = NULL; // translation between model types in header and here static int Nmodels = 0; @@ -135,4 +136,6 @@ models = NULL; Nmodels = 0; + psFree(modelClassLookupTable); + modelClassLookupTable = NULL; return; } @@ -193,2 +196,80 @@ } + +bool pmModelClassWriteHeader(psMetadata *header) +{ + psMetadataAddS32(header, PS_LIST_TAIL, "MTNUM", PS_META_REPLACE, "number of model types", Nmodels); + for (int i = 0; i < Nmodels; i++) { + char modelNameKey[16]; + char modelValKey[16]; + sprintf(modelNameKey, "MTNAM%02d", i); + sprintf(modelValKey, "MTVAL%02d", i); + psMetadataAddStr(header, PS_LIST_TAIL, modelNameKey, PS_META_REPLACE, "", models[i].name); + psMetadataAddS32(header, PS_LIST_TAIL, modelValKey, PS_META_REPLACE, "", i); + } + + return true; +} + +bool pmModelClassReadHeader(psMetadata *header) { + psFree(modelClassLookupTable); + + bool status; + int numHeaderModels = psMetadataLookupS32(&status, header, "MTNUM"); + if (!status) { + return false; + } + + psVector *inputTypes = psVectorAlloc(numHeaderModels, PS_TYPE_S32); + psVector *localTypes = psVectorAlloc(numHeaderModels, PS_TYPE_S32); + int max_val = -1; + for (int i = 0; i < numHeaderModels; i++) { + char modelNameKey[16]; + char modelValKey[16]; + sprintf(modelNameKey, "MTNAM%02d", i); + sprintf(modelValKey, "MTVAL%02d", i); + psString thisName = psMetadataLookupStr(&status, header, modelNameKey); + int thisVal = psMetadataLookupS32(&status, header, modelValKey); + if (thisVal > max_val) { + max_val = thisVal; + } + inputTypes->data.S32[i] = thisVal; + localTypes->data.S32[i] = pmModelClassGetType(thisName); + } + if (max_val < 0) { + psFree(inputTypes); + psFree(localTypes); + return false; + } + + modelClassLookupTable = psVectorAlloc(max_val + 1, PS_TYPE_S32); + psVectorInit(modelClassLookupTable, -1); + + for (int i = 0; i < numHeaderModels; i++) { + int thisVal = inputTypes->data.S32[i]; + int localVal = localTypes->data.S32[i]; + modelClassLookupTable->data.S32[thisVal] = localVal; + } + psFree(inputTypes); + psFree(localTypes); + + return true; +} + +pmModelType pmModelClassGetLocalType(pmModelType inputType) { + pmModelType localType = -1; + + if (modelClassLookupTable) { + if (inputType >= 0 && inputType < modelClassLookupTable->n) { + localType = modelClassLookupTable->data.S32[inputType]; + } + } else { + // no lookup table defined + // for backwards compatability if inputType refers to a defined model, return it + if (inputType >= 0 && pmModelClassGetName(inputType)) { + localType = inputType; + } + } + + return localType; +} Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmModelClass.h =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmModelClass.h (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmModelClass.h (revision 36480) @@ -76,4 +76,10 @@ void pmModelClassSetLimits(pmModelLimitsType type); +// write keywords to header definining the model type values used by this program +bool pmModelClassWriteHeader(psMetadata *header); +// create a lookup table for translating input model type values to local model type values +bool pmModelClassReadHeader(psMetadata *header); +// translate input model type value to local value +pmModelType pmModelClassGetLocalType(pmModelType inputType); /// @} Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO.c =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO.c (revision 36480) @@ -61,4 +61,5 @@ static bool pmReadoutReadXFIT(pmFPAfile *file, pmReadout *readout, char * exttype, psMetadata *hduHeader, psString xfitname, psArray *sources, long *sourceIndex); static bool pmReadoutReadXRAD(pmFPAfile *file, pmReadout *readout, char * exttype, psMetadata *hduHeader, psString xfitname, psArray *sources, long *sourceIndex); +static bool pmReadoutReadXGAL(pmFPAfile *file, pmReadout *readout, char * exttype, psMetadata *hduHeader, psString xfitname, psArray *sources, long *sourceIndex); // lookup the EXTNAME values used for table data and image header segments @@ -374,5 +375,5 @@ } \ if (xgalname) { \ - status &= pmSourcesWrite_##TYPE##_XGAL (file->fits, sources, xgalname, recipe); \ + status &= pmSourcesWrite_##TYPE##_XGAL (file->fits, readout, sources, xgalname, recipe); \ } \ } @@ -1036,5 +1037,5 @@ bool XFIT_OUTPUT = psMetadataLookupBool(&status, recipe, "EXTENDED_SOURCE_FITS"); bool XRAD_OUTPUT = psMetadataLookupBool(&status, recipe, "RADIAL_APERTURES"); - bool XGAL_OUTPUT = false; // psMetadataLookupBool(&status, recipe, "GALAXY_SHAPES"); + bool XGAL_OUTPUT = psMetadataLookupBool(&status, recipe, "GALAXY_SHAPES"); if (!pmSourceIOextnames(&headname, &dataname, &deteffname, @@ -1122,5 +1123,9 @@ long *sourceIndex = NULL; - if (XSRC_OUTPUT || XFIT_OUTPUT || XRAD_OUTPUT) { + if (XSRC_OUTPUT || XFIT_OUTPUT || XRAD_OUTPUT || XGAL_OUTPUT) { + // Build sourceIndex. Lookup table from source->seq to index in sources array. + // Consists of an array of length max(source->seq) + 1. + + // find maximum sequence number long seq_max = -1; for (long i = sources->n -1; i >= 0; i--) { @@ -1135,8 +1140,10 @@ } } + // allocate and initialize the index sourceIndex = psAlloc((seq_max + 1) * sizeof(long)); for (long i = 0; i < seq_max; i++) { sourceIndex[i] = -1; } + // populate the index for (long i = 0; i < sources->n; i++) { pmSource *source = sources->data[i]; @@ -1165,4 +1172,11 @@ psFree(xradname); } + if (XGAL_OUTPUT && xgalname) { + // a cmf file may have an XGAL extension, but it is not required + if (!pmReadoutReadXGAL(file, readout, exttype, hdu->header, xgalname, sources, sourceIndex)) { + // do anything? + } + psFree(xgalname); + } psFree(sourceIndex); @@ -1461,2 +1475,40 @@ return status; } +static bool pmReadoutReadXGAL(pmFPAfile *file, pmReadout *readout, char *exttype, psMetadata *hduHeader, psString xgalname, psArray *sources, long *sourceIndex) +{ + if (!psFitsMoveExtNameClean (file->fits, xgalname)) { + psTrace ("pmFPAfile", 1, "cannot find xgal extension %s in %s, skipping", xgalname, file->filename); + return false; + } + + psMetadata *tableHeader = psFitsReadHeader(NULL, file->fits); // The FITS header + if (!tableHeader) psAbort("cannot read table header"); + + char *xtension = psMetadataLookupStr (NULL, tableHeader, "XTENSION"); + if (!xtension) psAbort("cannot read table type"); + if (strcmp (xtension, "BINTABLE")) { + psFree(tableHeader); + psWarning ("no binary table in extension %s, skipping\n", xgalname); + return false; + } + +# define PM_SOURCES_READ_XGAL(NAME,TYPE) \ + if (!strcmp (exttype, NAME)) { \ + status = pmSourcesRead_##TYPE##_XGAL(file->fits, readout, hduHeader, tableHeader, sources, sourceIndex); \ + } + + bool status = false; + if (file->type == PM_FPA_FILE_CMF) { + PM_SOURCES_READ_XGAL("PS1_V1", CMF_PS1_V1); + PM_SOURCES_READ_XGAL("PS1_V2", CMF_PS1_V2); + PM_SOURCES_READ_XGAL("PS1_V3", CMF_PS1_V3); + PM_SOURCES_READ_XGAL("PS1_V4", CMF_PS1_V4); + PM_SOURCES_READ_XGAL("PS1_SV1", CMF_PS1_SV1); + PM_SOURCES_READ_XGAL("PS1_SV2", CMF_PS1_SV2); + PM_SOURCES_READ_XGAL("PS1_DV1", CMF_PS1_DV1); + PM_SOURCES_READ_XGAL("PS1_DV2", CMF_PS1_DV2); + PM_SOURCES_READ_XGAL("PS1_DV3", CMF_PS1_DV3); + } + psFree(tableHeader); + return status; +} Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO.h =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO.h (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO.h (revision 36480) @@ -21,9 +21,10 @@ bool pmSourcesWrite_##TYPE##_XFIT(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname); \ bool pmSourcesWrite_##TYPE##_XRAD(psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe); \ - bool pmSourcesWrite_##TYPE##_XGAL(psFits *fits, psArray *sources, char *extname, psMetadata *recipe); \ + bool pmSourcesWrite_##TYPE##_XGAL(psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe); \ psArray *pmSourcesRead_##TYPE (psFits *fits, psMetadata *header); \ bool pmSourcesRead_##TYPE##_XSRC (psFits *fits, pmReadout *readout, psMetadata *header, psMetadata *tableHeader, psArray *sources, long *index); \ bool pmSourcesRead_##TYPE##_XFIT (psFits *fits, pmReadout *readout, psMetadata *header, psMetadata *tableHeader, psArray *sources, long *index); \ bool pmSourcesRead_##TYPE##_XRAD (psFits *fits, pmReadout *readout, psMetadata *header, psMetadata *tableHeader, psArray *sources, long *index);\ + bool pmSourcesRead_##TYPE##_XGAL (psFits *fits, pmReadout *readout, psMetadata *header, psMetadata *tableHeader, psArray *sources, long *index);\ // All of these functions need to use the same API, even if not all elements are used in a specific case Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_CFF.c =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_CFF.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_CFF.c (revision 36480) @@ -65,4 +65,7 @@ int modelType = pmModelClassGetType ("PS_MODEL_GAUSS"); + // Read lookup table for model classes (if defined) + pmModelClassReadHeader(header); + char *PSF_NAME = psMetadataLookupStr (&status, header, "PSFMODEL"); if (PSF_NAME != NULL) { @@ -111,9 +114,10 @@ float theta = psMetadataLookupF32 (&status, row, "THETA"); - // XXX: we need to put a lookup table in the cff header to define the correspondence of the - // model type values in the cff with our models. (We want to use an interger for efficiency - // but the value for each model type is set on the organization of the the array in pmModelClass.c - // For now use the input values verbatim and trust the user that this is valid value int galaxyModelType = psMetadataLookupS32(&status, row, "MODEL_TYPE"); + if (status) { + galaxyModelType = pmModelClassGetLocalType(galaxyModelType); + } else { + galaxyModelType = -1; + } float Sindex = psMetadataLookupF32 (&status, row, "INDEX"); // Should this be PAR_07 not sersic index @@ -123,5 +127,6 @@ source->type = PM_SOURCE_TYPE_STAR; // XXX this should be added to the flags - // XXX we can set this in general, but for a specific image, we need to weed out SATSTARS + // XXX we can set this in general, but for a specific image, we need to weed out SATSTARS and + // stars that are masked if (psfStar) { source->tmpFlags |= PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; @@ -180,7 +185,5 @@ } - // XXX: should use < 0 as invalid galaxyModelType - - if (fitGalaxy && galaxyModelType > 0) { + if (fitGalaxy && galaxyModelType >= 0) { source->modelFits = psArrayAllocEmpty (1); pmModel *model = pmModelAlloc(galaxyModelType); @@ -235,4 +238,8 @@ PS_ASSERT(mdok, false); + // write the definition of the model class type values to the header + psMetadata *outputHeader = psMetadataAlloc(); + pmModelClassWriteHeader(outputHeader); + psArray *table = psArrayAllocEmpty(sources->n); @@ -246,7 +253,8 @@ psS32 modelType = 0; bool fitGalaxy = false; - bool psfStar = false; + bool psfStar = (source->mode & PM_SOURCE_MODE_PSFSTAR) ? true : false; psF32 sersicIndex = 0; - if (source->modelFits == NULL) { + // For now only perform galaxy fits on extended objects + if (source->modelEXT == NULL) { pmModel *model = source->modelPSF; if (model == NULL) continue; @@ -260,5 +268,4 @@ yPos = model->params->data.F32[PM_PAR_YPOS]; flux = source->psfFlux; - psfStar = (source->mode & PM_SOURCE_MODE_PSFSTAR) ? true : false; rMajor = 0; rMinor = 0; @@ -329,14 +336,15 @@ psArrayAdd(table, 100, row); + psFree(row); } - if (!psFitsWriteTable(fits, NULL, table, extname)) { + if (!psFitsWriteTable(fits, outputHeader, table, extname)) { psError(psErrorCodeLast(), false, "writing ext data %s\n", extname); psFree(table); - psFree(header); + psFree(outputHeader); return false; } psFree(table); - // psFree(header); + psFree(outputHeader); return true; Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_CMF.c.in =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_CMF.c.in (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_CMF.c.in (revision 36480) @@ -708,21 +708,9 @@ return false; } - // Find the source with this sequence number. - // XXX: I am assuming that sources is sorted in order of seq + // Find the source with this sequence number using the sourceIndex. long seq = psMetadataLookupU32 (&status, row, "IPP_IDET"); - pmSource *source = NULL; -#ifndef ASSUME_SORTED - long j = seq < sources->n ? seq : sources->n - 1; - for (; j >= 0; j--) { - source = sources->data[j]; - if (source->seq == seq) { - break; - } - } -#else long j = sourceIndex[seq]; psAssert(j >= 0 && j < sources->n, "invalid sourceIndex"); - source = sources->data[j]; -#endif + pmSource *source = sources->data[j]; if (!source) { psError(PS_ERR_UNKNOWN, false, "Failed to find source for row %ld sequence number %ld\n", i, seq); @@ -793,4 +781,6 @@ // create a header to hold the output data psMetadata *outhead = psMetadataAlloc (); + + pmModelClassWriteHeader(outhead); // write the links to the image header @@ -1034,4 +1024,7 @@ return false; } + // set up the lookup table to translate between input model types and output model types + // if not defined it is assumed that the tables are the same + pmModelClassReadHeader(tableHeader); for (long i = 0; i < numSources; i++) { @@ -1042,15 +1035,8 @@ return false; } - // Find the source with this sequence number. - // XXX: I am assuming that sources is sorted in order of seq. long seq = psMetadataLookupU32 (&status, row, "IPP_IDET"); - long j = seq < sources->n ? seq : sources->n - 1; - pmSource *source = NULL; - for (; j >= 0; j--) { - source = sources->data[j]; - if (source->seq == seq) { - break; - } - } + long j = sourceIndex[seq]; + psAssert(j >= 0 && j < sources->n, "invalid sourceIndex"); + pmSource *source = sources->data[j]; if (!source) { psError(PS_ERR_UNKNOWN, false, "Failed to find source for row %ld sequence number %ld\n", i, seq); @@ -1095,8 +1081,11 @@ // in the psf table. psS32 extModelType = psMetadataLookupS32(&status, row, "EXT_MODEL_TYPE"); - if (!status) { + if (status) { + // translate between the type value in xfit and values used by this program + extModelType = pmModelClassGetLocalType(extModelType); + } else { // older cmfs don't have this column extModelType = -1; - } + } psEllipseAxes axes; @@ -1339,15 +1328,8 @@ return false; } - // Find the source with this sequence number. - // XXX: I am assuming that sources is sorted in order of seq. long seq = psMetadataLookupU32 (&status, row, "IPP_IDET"); - long j = seq < sources->n ? seq : sources->n - 1; - pmSource *source = NULL; - for (; j >= 0; j--) { - source = sources->data[j]; - if (source->seq == seq) { - break; - } - } + long j = sourceIndex[seq]; + psAssert(j >= 0 && j < sources->n, "invalid sourceIndex"); + pmSource *source = sources->data[j]; if (!source) { psError(PS_ERR_UNKNOWN, false, "Failed to find source for row %ld sequence number %ld\n", i, seq); @@ -1407,5 +1389,5 @@ // XXX where should I record the number of columns?? -bool pmSourcesWrite_CMF_@CMFMODE@_XGAL (psFits *fits, psArray *sources, char *extname, psMetadata *recipe) +bool pmSourcesWrite_CMF_@CMFMODE@_XGAL (psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe) { bool status = false; @@ -1422,4 +1404,6 @@ // write the links to the image header psMetadataAddStr (outhead, PS_LIST_TAIL, "EXTNAME", PS_META_REPLACE, "galaxy table extension", extname); + + psMetadataAddStr (outhead, PS_LIST_TAIL, "HI", PS_META_REPLACE, "does this get through?", "THERE"); // let's write these out in S/N order @@ -1491,2 +1475,56 @@ } +bool pmSourcesRead_CMF_@CMFMODE@_XGAL(psFits *fits, pmReadout *readout, psMetadata *hduHeader, psMetadata *tableHeader, psArray *sources, long *sourceIndex) +{ + PS_ASSERT_PTR_NON_NULL(fits, false); + PS_ASSERT_PTR_NON_NULL(sources, false); + + bool status; + long numSources = psFitsTableSize(fits); // Number of sources in table + if (numSources == 0) { + psError(psErrorCodeLast(), false, "XGAL Table contains no entries\n"); + return false; + } + + for (long i = 0; i < numSources; i++) { + psMetadata *row = psFitsReadTableRow(fits, i); // Table row + if (!row) { + psError(psErrorCodeLast(), false, "Unable to read row %ld of sources", i); + psFree(row); + return false; + } + // Find the source with this sequence number. + // XXX: I am assuming that sources is sorted in order of seq + long seq = psMetadataLookupU32 (&status, row, "IPP_IDET"); + long j = sourceIndex[seq]; + psAssert(j >= 0 && j < sources->n, "invalid sourceIndex"); + + pmSource *source = sources->data[j]; + if (!source) { + psError(PS_ERR_UNKNOWN, false, "Failed to find source for row %ld sequence number %ld\n", i, seq); + psFree(row); + return false; + } + + psVector *Flux = psMetadataLookupVector(&status, row, "GAL_FLUX"); + psVector *dFlux = psMetadataLookupVector(&status, row, "GAL_FLUX_ERR"); + psVector *chisq = psMetadataLookupVector(&status, row, "GAL_CHISQ"); + + if (Flux && Flux->n > 0) { + psFree(source->galaxyFits); + source->galaxyFits = pmSourceGalaxyFitsAlloc(); + source->galaxyFits->nPix = psMetadataLookupF32(&status, row, "NPIX"); + + psFree(source->galaxyFits->Flux); + source->galaxyFits->Flux = psMemIncrRefCounter(Flux); + psFree(source->galaxyFits->dFlux); + source->galaxyFits->dFlux = psMemIncrRefCounter(dFlux); + psFree(source->galaxyFits->chisq); + source->galaxyFits->chisq = psMemIncrRefCounter(chisq); + } + + psFree(row); + } + + return true; +} Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_CAL_0.c =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_CAL_0.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_CAL_0.c (revision 36480) @@ -714,5 +714,5 @@ } -bool pmSourcesWrite_PS1_CAL_0_XGAL (psFits *fits, psArray *sources, char *extname, psMetadata *recipe) +bool pmSourcesWrite_PS1_CAL_0_XGAL (psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe) { return true; Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_DEV_0.c =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_DEV_0.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_DEV_0.c (revision 36480) @@ -256,6 +256,6 @@ } -bool pmSourcesWrite_PS1_DEV_0_XGAL(psFits *fits, psArray *sources, char *extname, psMetadata *recipe) -{ - return true; -} +bool pmSourcesWrite_PS1_DEV_0_XGAL(psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe) +{ + return true; +} Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_DEV_1.c =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_DEV_1.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_PS1_DEV_1.c (revision 36480) @@ -596,5 +596,5 @@ } -bool pmSourcesWrite_PS1_DEV_1_XGAL(psFits *fits, psArray *sources, char *extname, psMetadata *recipe) +bool pmSourcesWrite_PS1_DEV_1_XGAL(psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe) { return true; Index: branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_SMPDATA.c =================================================================== --- branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_SMPDATA.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psModules/src/objects/pmSourceIO_SMPDATA.c (revision 36480) @@ -226,5 +226,5 @@ } -bool pmSourcesWrite_SMPDATA_XGAL(psFits *fits, psArray *sources, char *extname, psMetadata *recipe) +bool pmSourcesWrite_SMPDATA_XGAL(psFits *fits, pmReadout *readout, psArray *sources, char *extname, psMetadata *recipe) { return true; Index: branches/eam_branches/ipp-20131211/psastro/src/psastroDataSave.c =================================================================== --- branches/eam_branches/ipp-20131211/psastro/src/psastroDataSave.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psastro/src/psastroDataSave.c (revision 36480) @@ -42,8 +42,9 @@ } - // de-activate all files except PSASTRO.OUTPUT + // de-activate all files except PSASTRO.OUTPUT, PSASTRO.OUT.ASTROM, and PSPHOT.OUTPUT.CFF pmFPAfileActivate (config->files, false, NULL); pmFPAfileActivate (config->files, true, "PSASTRO.OUTPUT"); pmFPAfileActivate (config->files, true, "PSASTRO.OUT.ASTROM"); + pmFPAfileActivate (config->files, true, "PSPHOT.OUTPUT.CFF"); pmFPAview *view = pmFPAviewAlloc (0); @@ -103,7 +104,8 @@ } - // activate all files except PSASTRO.OUTPUT + // activate all files except PSASTRO.OUTPUT, and PSPHOT.OUTPUT.CFF pmFPAfileActivate (config->files, true, NULL); pmFPAfileActivate (config->files, false, "PSASTRO.OUTPUT"); + pmFPAfileActivate (config->files, false, "PSPHOT.OUTPUT.CFF"); psFree (view); Index: branches/eam_branches/ipp-20131211/psastro/src/psastroDefineFiles.c =================================================================== --- branches/eam_branches/ipp-20131211/psastro/src/psastroDefineFiles.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psastro/src/psastroDefineFiles.c (revision 36480) @@ -100,4 +100,18 @@ } + bool writeCff = psMetadataLookupBool (&status, recipe, "PSASTRO.SAVE.CFF"); + if (writeCff) { + pmFPAfile *file = pmFPAfileDefineOutputFromFile (config, input, "PSPHOT.OUTPUT.CFF"); + if (!file) { + psError (PS_ERR_IO, false, "Can't find the output cff file definition"); + return NULL; + } + if (file->type != PM_FPA_FILE_CFF) { + psError(PS_ERR_IO, true, "%s is not of type %s", "PSPHOT.OUTPUT.CFF", pmFPAfileStringFromType (PM_FPA_FILE_CFF)); + return NULL; + } + file->save = true; + } + # if (0) Index: branches/eam_branches/ipp-20131211/psphot/src/Makefile.am =================================================================== --- branches/eam_branches/ipp-20131211/psphot/src/Makefile.am (revision 36479) +++ branches/eam_branches/ipp-20131211/psphot/src/Makefile.am (revision 36480) @@ -25,5 +25,5 @@ libpsphot_la_LDFLAGS = $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) -bin_PROGRAMS = psphot psphotForced psphotFullForce psphotMinimal psphotMakePSF psphotStack psphotModelTest psmakecff +bin_PROGRAMS = psphot psphotForced psphotFullForce psphotFullForceSummary psphotMinimal psphotMakePSF psphotStack psphotModelTest psmakecff # bin_PROGRAMS = psphotPetrosianStudy psphotTest psphotMomentsStudy @@ -39,4 +39,8 @@ psphotFullForce_LDFLAGS = $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) psphotFullForce_LDADD = libpsphot.la + +psphotFullForceSummary_CFLAGS = $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) +psphotFullForceSummary_LDFLAGS = $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) +psphotFullForceSummary_LDADD = libpsphot.la psphotMinimal_CFLAGS = $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) @@ -99,4 +103,9 @@ psphotMosaicChip.c \ psphotCleanup.c + +# combine full force results from several inputs +psphotFullForceSummary_SOURCES = \ + psphotFullForceSummary.c \ + psphotFullForceSummaryReadout.c # forced photometry of specified positions given a specified psf Index: branches/eam_branches/ipp-20131211/psphot/src/psmakecff.c =================================================================== --- branches/eam_branches/ipp-20131211/psphot/src/psmakecff.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psphot/src/psmakecff.c (revision 36480) @@ -2,4 +2,7 @@ # include # endif + +// psmakecff : A program to make read a cmf file and write a cff file +// The real work is done in psModules. #include @@ -8,4 +11,5 @@ #include "psphot.h" +// For simplicilty, this program's (simple) functions are all contained in this file. static pmConfig* psmakecffArguments(int, char**); static bool psmakecffParseCamera(pmConfig *); @@ -25,5 +29,4 @@ // psphotVersionPrint(); - // load input data (config and images (signal, noise, mask) if (!psmakecffParseCamera (config)) { psErrorStackPrint(stderr, "Error setting up the camera\n"); @@ -31,5 +34,4 @@ } - // call psphot for each readout if (!psmakecffImageLoop (config)) { psErrorStackPrint(stderr, "Error in the psphot image loop\n"); @@ -37,8 +39,4 @@ } -// psLogMsg ("psphot", PS_LOG_WARN, "complete psphot run: %f sec\n", psTimerMark ("complete")); - -// psErrorCode exit_status = psphotGetExitStatus(); -// psphotCleanup (config); exit (0); } @@ -73,4 +71,5 @@ return config; } + static bool psmakecffParseCamera(pmConfig *config) { bool status = false; @@ -127,28 +126,24 @@ psAssert (recipe, "missing recipe?"); -// psImageMaskType maskTest = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // for psphot, we force data to be read at the chip level while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { - psLogMsg ("psmakecmf", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + psLogMsg ("psmakecff", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); if (! chip->process || ! chip->file_exists) { continue; } -#ifdef notdef - pmFPAfileActivate (config->files, false, NULL); - pmFPAfileActivate (config->files, true, "PSPHOT.LOAD"); -#endif if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psmakecff."); // there is now only a single chip (multiple readouts?). loop over it and process while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { - psLogMsg ("psmakecmf", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + psLogMsg ("psmakecff", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); // process each of the readouts while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { - psLogMsg ("psmakecmf", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + psLogMsg ("psmakecff", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); if (! readout->data_exists) { continue; } pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell); if (hdu && hdu != lastHDU) { + // XXX: probably should do this // psphotVersionHeaderFull(hdu->header); lastHDU = hdu; @@ -157,12 +152,9 @@ } - // Defer output and closing of files until we've (possibly) done the NFrames analysis below -// pmFPAfileActivate (config->files, false, NULL); - if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for Chip in psmakecff."); } if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for FPA in psphot."); - // fail if we failed to handle an error + // fail if we encountered an unhandled error if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); Index: branches/eam_branches/ipp-20131211/psphot/src/psphot.h =================================================================== --- branches/eam_branches/ipp-20131211/psphot/src/psphot.h (revision 36479) +++ branches/eam_branches/ipp-20131211/psphot/src/psphot.h (revision 36480) @@ -361,4 +361,6 @@ bool psphotModelTestReadout(pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotFullForceSummaryReadout (pmConfig * config, const pmFPAview *view); + int psphotFileruleCount(const pmConfig *config, const char *filerule); bool psphotFileruleCountSet(const pmConfig *config, const char *filerule, int num); Index: branches/eam_branches/ipp-20131211/psphot/src/psphotChooseAnalysisOptions.c =================================================================== --- branches/eam_branches/ipp-20131211/psphot/src/psphotChooseAnalysisOptions.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psphot/src/psphotChooseAnalysisOptions.c (revision 36480) @@ -60,5 +60,5 @@ } -bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos); +static bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos); // this function use an internal flag to mark sources which have already been measured @@ -121,9 +121,21 @@ psSphereRot *toGal = NULL; float GAL_LIMIT = 0.0; + float GAL_LIMIT_BULGE = 0.0; + float GAL_LIMIT_SIGMA2 = 0.0; bool useGAL_LIMIT = psMetadataLookupBool(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.USE"); assert (status); if (useGAL_LIMIT) { + toGal = psSphereRotICRSToGalactic(); GAL_LIMIT = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT"); - toGal = psSphereRotICRSToGalactic(); + assert (status); + GAL_LIMIT = DEG_TO_RAD(GAL_LIMIT); + GAL_LIMIT_BULGE = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE"); + assert (status); + GAL_LIMIT_BULGE = DEG_TO_RAD(GAL_LIMIT_BULGE); + float GAL_LIMIT_SIGMA = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE.SIGMA"); + assert (status); + assert(GAL_LIMIT_SIGMA > 0); + GAL_LIMIT_SIGMA = DEG_TO_RAD(GAL_LIMIT_SIGMA); + GAL_LIMIT_SIGMA2 = GAL_LIMIT_SIGMA * GAL_LIMIT_SIGMA; } @@ -135,4 +147,5 @@ } + float petroFluxLim = NAN; float extFitFluxLim = NAN; @@ -140,4 +153,6 @@ float extFitMagLimDefault = NAN; float extFitMagLim = NAN; + float petroMagLimDefault = NAN; + float petroMagLim = NAN; // match to the given filter @@ -156,17 +171,26 @@ // find a matching filter or default to 'any' if (!strcasecmp (thisFilter, "any")) { - psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT"); - psAssert(extFitMagLimStr, "missing MAG.LIMIT"); + psString petroMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO"); + psAssert(petroMagLimStr, "missing MAG.LIMIT.PETRO"); + petroMagLimDefault = atof (petroMagLimStr); + + psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT"); + psAssert(extFitMagLimStr, "missing MAG.LIMIT.EXTFIT"); extFitMagLimDefault = atof (extFitMagLimStr); } - // find a matching filter or default to 'any' if (!strcasecmp (thisFilter, filterID)) { - psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT"); - psAssert(extFitMagLimStr, "missing MAG.LIMIT"); + psString petroMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO"); + psAssert(petroMagLimStr, "missing MAG.LIMIT.PETRO"); + petroMagLim = atof (petroMagLimStr); + + psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT"); + psAssert(extFitMagLimStr, "missing MAG.LIMIT.EXTFIT"); extFitMagLim = atof (extFitMagLimStr); break; } } + psFree(iter); + if (!isfinite (petroMagLim)) petroMagLim = petroMagLimDefault; if (!isfinite (extFitMagLim)) extFitMagLim = extFitMagLimDefault; @@ -185,4 +209,5 @@ psAssert (status, "missing FPA.ZP?"); + petroFluxLim = exptime * pow (10.0, 0.4*(zeropt - petroMagLim)); extFitFluxLim = exptime * pow (10.0, 0.4*(zeropt - extFitMagLim)); } @@ -225,18 +250,27 @@ psSphere ptGal, ptSky; GetGalacticCoords (&ptGal, &ptSky, toGal, chip, source->peak->xf, source->peak->yf); - if (fabs(ptGal.d) < GAL_LIMIT) continue; + float l = ptGal.r; + float b_min = GAL_LIMIT + GAL_LIMIT_BULGE * exp(-0.5*(l*l/GAL_LIMIT_SIGMA2)); + if (fabs(ptGal.d) < b_min) continue; // include an exception for low density skycells below the limit? } // for petro and extFit, we will either use the mag limits or the S/N + if (doPetrosian) { + if (isfinite(petroFluxLim)) { + if (source->moments->KronFlux > petroFluxLim) { + source->tmpFlags |= PM_SOURCE_TMPF_PETRO; + } + } else if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) { + source->tmpFlags |= PM_SOURCE_TMPF_PETRO; + } + } if (isfinite(extFitFluxLim)) { if (source->moments->KronFlux > extFitFluxLim) { source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT; - if (doPetrosian) source->tmpFlags |= PM_SOURCE_TMPF_PETRO; } } else { if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) { source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT; - if (doPetrosian) source->tmpFlags |= PM_SOURCE_TMPF_PETRO; } } @@ -244,4 +278,6 @@ psLogMsg ("psphot.options", PS_LOG_WARN, "choose analysis options for %ld sources: %f sec\n", sources->n, psTimerMark ("psphot.options")); + + psFree(toGal); return true; @@ -287,9 +323,21 @@ psSphereRot *toGal = NULL; float GAL_LIMIT = 0.0; + float GAL_LIMIT_BULGE = 0.0; + float GAL_LIMIT_SIGMA2 = 0.0; bool useGAL_LIMIT = psMetadataLookupBool(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.USE"); assert (status); if (useGAL_LIMIT) { + toGal = psSphereRotICRSToGalactic(); GAL_LIMIT = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT"); - toGal = psSphereRotICRSToGalactic(); + assert (status); + GAL_LIMIT = DEG_TO_RAD(GAL_LIMIT); + GAL_LIMIT_BULGE = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE"); + assert (status); + GAL_LIMIT_BULGE = DEG_TO_RAD(GAL_LIMIT_BULGE); + float GAL_LIMIT_SIGMA = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE.SIGMA"); + assert (status); + assert(GAL_LIMIT_SIGMA > 0); + GAL_LIMIT_SIGMA = DEG_TO_RAD(GAL_LIMIT_SIGMA); + GAL_LIMIT_SIGMA2 = GAL_LIMIT_SIGMA * GAL_LIMIT_SIGMA; } @@ -365,9 +413,13 @@ // find extFitFluxLim->data.F32[i] for i == image number psVector *extFitFluxLim = NULL; + psVector *petroFluxLim = NULL; if (magLimits) { extFitFluxLim = psVectorAlloc (inputFilters->n, PS_TYPE_F32); + petroFluxLim = psVectorAlloc (inputFilters->n, PS_TYPE_F32); psVectorInit (extFitFluxLim, NAN); + psVectorInit (petroFluxLim, NAN); float extFitMagLimDefault = NAN; + float petroMagLimDefault = NAN; // match mag limits (flux limits) to the filters @@ -383,7 +435,11 @@ // save the default value to assign to unset filters if (!strcasecmp (thisFilter, "any")) { - psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT"); - psAssert(magString, "missing MAG.LIMIT"); + psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT"); + psAssert(magString, "missing MAG.LIMIT.EXTFIT"); extFitMagLimDefault = atof (magString); + + magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO"); + psAssert(magString, "missing MAG.LIMIT.PETRO"); + petroMagLimDefault = atof (magString); continue; } @@ -393,7 +449,12 @@ if (i == chisqNum) continue; if (!strcasecmp (thisFilter, inputFilters->data[i])) { - psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT"); - psAssert(magString, "missing MAG.LIMIT"); + psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO"); + psAssert(magString, "missing MAG.LIMIT.PETRO"); float magvalue = atof (magString); + petroFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - magvalue)); + + magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT"); + psAssert(magString, "missing MAG.LIMIT.EXTFIT"); + magvalue = atof (magString); extFitFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - magvalue)); break; @@ -401,9 +462,14 @@ } } + psFree(iter); for (int i = 0; i < num; i++) { if (i == chisqNum) continue; - if (isfinite(extFitFluxLim->data.F32[i])) continue; - extFitFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - extFitMagLimDefault)); + if (!isfinite(petroFluxLim->data.F32[i])) { + petroFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - petroMagLimDefault)); + } + if (!isfinite(extFitFluxLim->data.F32[i])) { + extFitFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - extFitMagLimDefault)); + } } } @@ -420,5 +486,6 @@ bool doObjectRadial = false; bool doObjectExtFit = false; - for (int j = 0; !doObjectExtFit && !doObjectRadial && (j < object->sources->n); j++) { + bool doObjectPetrosian = false; + for (int j = 0; !doObjectExtFit && !doObjectRadial && !doObjectPetrosian && (j < object->sources->n); j++) { pmSource *source = object->sources->data[j]; @@ -451,4 +518,5 @@ if (extFitAll) { doObjectExtFit = true; + doObjectPetrosian = doPetrosian; continue; } @@ -465,9 +533,24 @@ psSphere ptGal, ptSky; GetGalacticCoords (&ptGal, &ptSky, toGal, chips->data[imageID], source->peak->xf, source->peak->yf); - if (fabs(ptGal.d) < GAL_LIMIT) continue; - // include an exception for low density skycells below the limit? - } - - float fluxLim = extFitFluxLim ? extFitFluxLim->data.F32[imageID] : NAN; + float l = ptGal.r; + float b_min = GAL_LIMIT + GAL_LIMIT_BULGE * exp(-0.5*(l*l/GAL_LIMIT_SIGMA2)); + if (fabs(ptGal.d) < b_min) continue; + } + + float fluxLim = NAN; + // for petro and extFit, we will either use the mag limits or the S/N + if (doPetrosian) { + fluxLim = petroFluxLim ? petroFluxLim->data.F32[imageID] : NAN; + if (isfinite(fluxLim)) { + if (source->moments->KronFlux > extFitFluxLim->data.F32[imageID]) { + doObjectPetrosian = true; + } + } else { + if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) { + doObjectPetrosian = true; + } + } + } + fluxLim = extFitFluxLim ? extFitFluxLim->data.F32[imageID] : NAN; // for petro and extFit, we will either use the mag limits or the S/N if (isfinite(fluxLim)) { @@ -503,9 +586,10 @@ if (source->modelPSF == NULL) continue; - // Do the fits if the recipe requests we do extended source fits to everything if (doObjectExtFit) { source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT; - if (doPetrosian) source->tmpFlags |= PM_SOURCE_TMPF_PETRO; - } + } + if (doObjectPetrosian) { + source->tmpFlags |= PM_SOURCE_TMPF_PETRO; + } // Do the fits if the recipe requests we do extended source fits to everything @@ -522,8 +606,10 @@ psLogMsg ("psphot.options", PS_LOG_WARN, "choose analysis options for %ld objects: %f sec\n", objects->n, psTimerMark ("psphot.options")); + psFree(toGal); + return true; } -bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos) { +static bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos) { pmFPA *fpa = chip->parent; @@ -543,4 +629,9 @@ psSphereRotApply (ptGal, toGal, ptSky); + // psSphereRotApply insures that 0 < r < 2PI. We want -PI < b <= PI + if (ptGal->r > M_PI) { + ptGal->r -= 2.0 * M_PI; + } + return true; Index: branches/eam_branches/ipp-20131211/psphot/src/psphotFullForceSummary.c =================================================================== --- branches/eam_branches/ipp-20131211/psphot/src/psphotFullForceSummary.c (revision 36480) +++ branches/eam_branches/ipp-20131211/psphot/src/psphotFullForceSummary.c (revision 36480) @@ -0,0 +1,266 @@ +# ifdef HAVE_CONFIG_H +# include +# endif + +#include +#include +#include +#include "psphot.h" + +// For simplicilty, this program's (simple) functions are all contained in this file. +static pmConfig* psphotFullForceSummaryArguments(int, char**); +static bool psphotFullForceSummaryParseCamera(pmConfig *); +static bool psphotFullForceSummaryImageLoop(pmConfig*); + +int main (int argc, char **argv) { + + psMemInit(); + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotFullForceSummaryArguments (argc, argv); + assert(config); + +// psphotVersionPrint(); + + if (!psphotFullForceSummaryParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (1); + } + + if (!psphotFullForceSummaryImageLoop (config)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit(1); + } + + // XXX:check for memory leaks + exit (0); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status + +void usage() { + fprintf(stderr, "usage: psphotFullForceSummary -inputs \n"); + exit (1); +} + +static pmConfig* psphotFullForceSummaryArguments(int argc, char **argv) { + + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + int N; + + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + if ((N = psArgumentGet(argc, argv, "-input"))) { + if (argc <= N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + usage(); + } + psArgumentRemove(N, &argc, argv); + + unsigned int numBad = 0; // Number of bad lines + psMetadata *inputs = psMetadataConfigRead(NULL, &numBad, argv[N], false); // Input file info + if (!inputs || numBad > 0) { + psErrorStackPrint(stderr, "Unable to cleanly read MDC file with inputs."); + exit(PS_EXIT_CONFIG_ERROR); + } + psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "INPUTS", 0, "Metadata with input details", inputs); + psFree(inputs); + + psArgumentRemove(N, &argc, argv); + } else { + psErrorStackPrint(stderr, "-input must be supplied."); + usage(); + } + + if (argc < 2) { + psErrorStackPrint(stderr, "Output is required."); + usage(); + } + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", PS_DATA_STRING, "", argv[1]); + + return config; +} + +static bool psphotFullForceSummaryParseCamera(pmConfig *config) { + bool status = false; + + psMetadata *inputs = psMetadataLookupMetadata(&status, config->arguments, "INPUTS"); // The inputs info + if (!inputs) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find inputs."); + return false; + } + + int nInputs = inputs->list->n; + + for (int i = 0; i < nInputs; i++) { + psMetadataItem *item = psMetadataGet(inputs, i); + + if (item->type != PS_DATA_STRING) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Component %s of the input metadata is not of type STRING", item->name); + return false; + + } + psString sourcesFilename = item->data.str; + + psArray *dummy = psArrayAlloc(1); // dummy array of filenames + dummy->data[0] = psStringCopy(sourcesFilename); + + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "FILENAMES", PS_META_REPLACE, + "Filenames for file rule definition", dummy); + psFree(dummy); + + bool found = false; + pmFPAfile *file = pmFPAfileDefineFromArgs(&found, config, "PSPHOT.INPUT.CMF", "FILENAMES"); + if (!file || !found) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file %s from %s", "PSPHOT.INPUT.CMF", sourcesFilename); + return false; + } + if (file->type != PM_FPA_FILE_CMF) { + psError(PS_ERR_IO, true, "%s is not of type %s", sourcesFilename, pmFPAfileStringFromType(PM_FPA_FILE_CMF)); + return false; + } + } + psMetadataRemoveKey(config->arguments, "FILENAMES"); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.CMF.NUM", PS_META_REPLACE, "number of inputs", + nInputs); + + + pmFPA *outputFPA = pmFPAConstruct(config->camera, config->cameraName); + if (!outputFPA) { + psError(psErrorCodeLast(), false, "Unable to construct an FPA from camera configuration."); + return false; + } + pmFPAfile *output = pmFPAfileDefineOutput(config, outputFPA, "PSPHOT.FULLFORCE.OUTPUT"); + psFree(outputFPA); // Drop reference + if (!output) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PSPHOT.FULLFORCE.OUTPUT")); + return false; + } + if (output->type != PM_FPA_FILE_CMF) { + psError(PSPHOT_ERR_CONFIG, true, "PSPHOT.FULLFORCE.OUTPUT is not of type CMF"); + return false; + } + output->save = true; + + return true; +} + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ + } + +bool psphotFullForceSummaryImageLoop (pmConfig *config) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT.CMF"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + + pmFPAfile *output = psMetadataLookupPtr (&status, config->files, "PSPHOT.FULLFORCE.OUTPUT"); + if (!output) { + psError(PSPHOT_ERR_PROG, false, "Can't find output data!"); + return false; + } + + pmFPAview *view = pmFPAviewAlloc (0); + if (!pmFPAAddSourceFromView(output->fpa, view, output->format)) { + psError(PS_ERR_UNKNOWN, false, "Unable to insert HDU into FPA for writing.\n"); + psFree(view); + return NULL; + } + + // files associated with the science image + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa in psphot."); + + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { +// psLogMsg ("psphotFullForceSummary", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphotFullForceSummary."); + + // We read the WCS from the first input + { + pmHDU *hduLow = pmHDUGetLowest(input->fpa, chip, NULL); + if (hduLow && !pmAstromReadWCS(input->fpa, chip, hduLow->header, 1.0)) { + psWarning("Unable to read WCS astrometry from header."); + psErrorClear(); + pmHDU *hduHigh = pmHDUGetHighest(input->fpa, chip, NULL); + if (hduHigh && hduHigh != hduLow && + !pmAstromReadWCS(input->fpa, chip, hduHigh->header, 1.0)) { + psWarning("Unable to read WCS astrometry from primary header."); + psErrorClear(); + } + } + } + + // Copy the transformations from the input + output->fpa->fromTPA = psMemIncrRefCounter(input->fpa->fromTPA); + output->fpa->toTPA = psMemIncrRefCounter(input->fpa->toTPA); + output->fpa->toSky = psMemIncrRefCounter(input->fpa->toSky); + pmChip *outputChip = pmFPAviewThisChip(view, output->fpa); + outputChip->toFPA = psMemIncrRefCounter(chip->toFPA); + outputChip->fromFPA = psMemIncrRefCounter(chip->fromFPA); + if (output->fpa->hdu->header == NULL) { + output->fpa->hdu->header = psMetadataAlloc(); + } + // XXX: how come psphot and psphotStack don't have to do this? + if (!pmAstromWriteWCS(output->fpa->hdu->header, output->fpa, outputChip, 0.001)) { + ESCAPE("failure to copy WCS to header"); + } + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + // psLogMsg ("psphotFullForceSummary", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { +// psLogMsg ("psphotFullForceSummary", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + if (!psphotFullForceSummaryReadout(config, view)) { + ESCAPE ("failure in psphotFullForceSummaryReadout"); + } + } + } + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for Chip in psphotFullForceSummary."); + } + + + // If these keywords are not set we get a warning message on output. Since we are not copying the input header + // and do not have an image pmFPA stuff doesn't have values for these + // XXX: What other keywords and concepts should be set (or copied)? + int numCols = psMetadataLookupS32(&status, input->fpa->hdu->header, "IMNAXIS1"); + int numRows = psMetadataLookupS32(&status, input->fpa->hdu->header, "IMNAXIS2"); + psMetadataAddS32(output->fpa->hdu->header, PS_LIST_TAIL, "IMNAXIS1", PS_META_REPLACE, "", numCols); + psMetadataAddS32(output->fpa->hdu->header, PS_LIST_TAIL, "IMNAXIS2", PS_META_REPLACE, "", numRows); + + // XXX: Also add psphot version information + + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for FPA in psphot."); + + // fail if we encountered an unhandled error + if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); + + psFree (view); + + return true; +} Index: branches/eam_branches/ipp-20131211/psphot/src/psphotFullForceSummaryReadout.c =================================================================== --- branches/eam_branches/ipp-20131211/psphot/src/psphotFullForceSummaryReadout.c (revision 36480) +++ branches/eam_branches/ipp-20131211/psphot/src/psphotFullForceSummaryReadout.c (revision 36480) @@ -0,0 +1,393 @@ +#include "psphotInternal.h" + + +typedef struct { + int numTrials; + psF32 fRmajorMin; + psF32 fRmajorMax; + psF32 fRmajorDel; + psF32 fRminorMin; + psF32 fRminorMax; + psF32 fRminorDel; + psVector *rMajor; + psVector *rMinor; + psArray *zeroPt; + psArray *exptime; +} galaxyShapeOptions; + +static pmSource *psphotFullForceSummarizeObject(pmConfig *config, pmPhotObj *obj, psVector *fluxScaleFactor, galaxyShapeOptions *options); +static pmPhotObj *findObjectForSource(psArray **pObjects, pmSource *source); + +static bool setOptions(galaxyShapeOptions *options, pmReadout *readout, psMetadata *recipe, bool saveVectors); +static bool checkOptions(galaxyShapeOptions *options, pmReadout *readout, psMetadata *recipe); + + +bool psphotFullForceSummaryReadout (pmConfig *config, const pmFPAview *view) { + + bool status; + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, "PSPHOT.INPUT.CMF"); + + pmFPAfile *output = psMetadataLookupPtr (&status, config->files, "PSPHOT.FULLFORCE.OUTPUT"); + pmCell *outputCell = pmFPAviewThisCell(view, output->fpa); + pmReadout *outputReadout = pmFPAviewThisReadout(view, output->fpa); + if (!outputReadout) { + outputReadout = pmReadoutAlloc(outputCell); + } + + // Get the exposure parameters for the output from recipe and set them on the output + psF32 outputZeroPoint = psMetadataLookupF32(&status, recipe, "PSPHOT.FULLFORCE.ZERO_PT"); + psF32 outputExptime = psMetadataLookupF32(&status, recipe, "PSPHOT.FULLFORCE.EXPTIME"); + + psMetadataAddF32(output->fpa->concepts, PS_LIST_TAIL, "FPA.ZP", PS_META_REPLACE, "Magnitude zero point", + outputZeroPoint); + psMetadataAddF32(outputCell->concepts, PS_LIST_TAIL, "CELL.EXPOSURE", PS_META_REPLACE, "Exposure time (sec)", + outputExptime); + // don't think this one matters +// psMetadataAddF32(output->fpa->concepts, PS_LIST_TAIL, "FPA.EXPOSURE", PS_META_REPLACE, "Exposure time (sec)", + // outputExptime); + + // Create objects from the various input's sources + // loop over the available readouts +// psArray *readouts = psArrayAlloc(num); + psVector *fluxScaleFactor = psVectorAlloc(num, PS_TYPE_F32); + galaxyShapeOptions options; + psArray *objects = NULL; + for (int index = 0; index < num; index++) { + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PSPHOT.INPUT.CMF", index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (index == 0) { + // Get the galaxy shape recipe values, from the analysis if present + // or from the recipe if not + if (!setOptions(&options, readout, recipe, true)) { + psError (PS_ERR_UNKNOWN, false, "problem determining galaxy shape options."); + return false; + } + } else { + // Make sure that this input was created with the same galaxy shapes recipe + if (!checkOptions(&options, readout, recipe)) { + psError (PS_ERR_UNKNOWN, false, "galaxy shape options do not match for input %d", index); + return false; + } + } + + // look up zero point + psF32 zero_point = psMetadataLookupF32(&status, readout->parent->parent->parent->concepts, "FPA.ZP"); + psF32 exptime = psMetadataLookupF32(&status, readout->parent->parent->parent->concepts, "FPA.EXPOSURE"); + + psF32 scaleFactor = pow(10, 0.4 * (outputZeroPoint - zero_point)) * outputExptime / exptime; + fluxScaleFactor->data.F32[index] = scaleFactor; + +// readouts->data[index] = readout; + + // find detections + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + sources = psArraySort (sources, pmSourceSortBySeq); + + if (objects == NULL) { + pmSource *lastSource = sources->data[sources->n - 1]; + psAssert(lastSource, "last source is null!"); + objects = psArrayAlloc(lastSource->seq + 1); + } + + for (int i=0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + source->imageID = index; + findObjectForSource(&objects, source); + } + } + + pmDetections *outputDetections = pmDetectionsAlloc(); + if (!psMetadataAddPtr (outputReadout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", + PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", outputDetections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + + psArray *outputSources = psArrayAllocEmpty (objects->n); + + // Loop over objects and compute the summaries + long nObjects = 0; + long nSources = 0; + for (int i=0 ; in; i++) { + pmPhotObj *obj = objects->data[i]; + if (!obj) continue; + + ++nObjects; + pmSource *source = psphotFullForceSummarizeObject(config, obj, fluxScaleFactor, &options); + if (source) { + psArrayAdd (outputSources, 100, source); + ++nSources; + } + } + + psLogMsg("psphot", PS_LOG_INFO, "constructed %ld output sources, from %ld objects.", nSources, nObjects); + + psFree(fluxScaleFactor); + + if (nSources) { + // We have data + outputDetections->allSources = outputSources; + outputReadout->data_exists = true; + outputReadout->parent->data_exists = true; + outputReadout->parent->parent->data_exists = true; + } else { + // XXX: tooo set a quality or fault code + return false; + } + + return true; +} + +static pmPhotObj *findObjectForSource(psArray **pObjects, pmSource *source) { + int seq = source->seq; + + psArray *objects = *pObjects; + if (seq >= objects->n) { + // We need to expand the object array. Kind of suprising. + objects = *pObjects = psArrayRealloc(objects, seq+1); + } + + // Look up object for this seq + pmPhotObj *obj = objects->data[seq]; + if (!obj) { + // not found allocate one + obj = pmPhotObjAlloc(); + objects->data[seq] = obj; + } + pmPhotObjAddSource(obj, source); + + return obj; +} + +static pmSource *psphotFullForceSummarizeObject(pmConfig *config, pmPhotObj *obj, psVector *fluxScaleFactor, galaxyShapeOptions *options) { + + pmSource *outSrc = NULL; + + psVector *sumWeightedFlux = NULL; + psVector *sumInvSig2 = NULL; + psVector *numerator = NULL; +// psVector *tmp = NULL; + psF32 totalNPix = 0; + long vectorLength = 0; + + for (int i=0; i < obj->sources->n; i++) { + pmSource *source = obj->sources->data[i]; + + // XXX: get cut from recipe + if (source->pixWeightNotPoor < .9) continue; + + // For now just start the output source as a copy of the first input source that makes cuts + if (!outSrc) { + outSrc = pmSourceCopy(source); + // This copies + // the peak + // the moments which are mostly nan except for Mrf Mx, My, and some of the kron parameters + // + // type, mode, flags + // magnitudes + outSrc->imageID = 0; + outSrc->seq = source->seq; + + if (source->modelPSF) { + outSrc->modelPSF = psMemIncrRefCounter(source->modelPSF); + } + if (source->extpars) { + outSrc->extpars = psMemIncrRefCounter(source->extpars); + } + if (source->modelEXT) { + outSrc->modelEXT = psMemIncrRefCounter(source->modelEXT); + } + if (source->modelFits) { + outSrc->modelFits = psMemIncrRefCounter(source->modelFits); + } + } + + if (source->galaxyFits && isfinite(source->galaxyFits->nPix) && source->galaxyFits->chisq->n) { + if (numerator == NULL) { + vectorLength = source->galaxyFits->chisq->n; + // tmp = psVectorAlloc(vectorLength, PS_TYPE_F32); + sumWeightedFlux = psVectorAlloc(vectorLength, PS_TYPE_F32); + psVectorInit(sumWeightedFlux, 0.0); + sumInvSig2 = psVectorAlloc(vectorLength, PS_TYPE_F32); + psVectorInit(sumInvSig2, 0.0); + numerator = psVectorAlloc(vectorLength, PS_TYPE_F32); + psVectorInit(numerator, 0.0); + } + + psAssert(vectorLength == options->numTrials, "length of chisq vector %ld does not match options %d", + vectorLength, options->numTrials); + psAssert(source->galaxyFits->chisq->n == vectorLength, "length of chisq vectors do not match %ld %ld", + source->galaxyFits->chisq->n, vectorLength); + + + psF32 scaleFactor = fluxScaleFactor->data.F32[source->imageID]; + + for (int k = 0; k < vectorLength; k++) { + + psF32 chisq = source->galaxyFits->chisq->data.F32[k]; + psF32 flux = source->galaxyFits->Flux->data.F32[k] * scaleFactor; + psF32 dFlux = source->galaxyFits->dFlux->data.F32[k] * scaleFactor; + // for chisq + // Numerator = sum( nPix[i] * chisq[i] ) + // denominator will be sum( nPix ) + + +// tmp = (psVector *) psBinaryOp(tmp, + // source->galaxyFits->chisq, "*", psScalarAlloc(source->galaxyFits->nPix, PS_TYPE_F32)); +// numerator = (psVector *) psBinaryOp(numerator, numerator, "+", tmp); + + numerator->data.F32[k] += chisq * source->galaxyFits->nPix; + totalNPix += source->galaxyFits->nPix; + + // sumInvSig2 = sum( 1 / dFlux**2 ) +// tmp = (psVector *) psBinaryOp(tmp, source->galaxyFits->dFlux, "*", source->galaxyFits->dFlux); +// tmp = (psVector *) psBinaryOp(tmp, psScalarAlloc(1.0, PS_TYPE_F32), "/", tmp); +// sumInvSig2 = (psVector *) psBinaryOp(sumInvSig2, sumInvSig2, "+", tmp); +// + psF32 invSig2 = 1.0 / (dFlux * dFlux); + sumInvSig2->data.F32[k] += invSig2; + + // sumWeightedFlux = sum ( Flux / dFlux**2 ) +// tmp = (psVector *) psBinaryOp(tmp, source->galaxyFits->Flux, "*", tmp); +// sumWeightedFlux = (psVector *) psBinaryOp(sumWeightedFlux, sumWeightedFlux, "+", tmp); + sumWeightedFlux->data.F32[k] += flux * invSig2; + } + } + } + + if (outSrc) { + if (vectorLength) { + outSrc->galaxyFits = pmSourceGalaxyFitsAlloc(); + outSrc->galaxyFits->nPix = totalNPix; + outSrc->galaxyFits->Flux = psVectorRecycle(outSrc->galaxyFits->Flux, vectorLength, PS_TYPE_F32); + outSrc->galaxyFits->dFlux = psVectorRecycle(outSrc->galaxyFits->dFlux, vectorLength, PS_TYPE_F32); + outSrc->galaxyFits->chisq = psVectorRecycle(outSrc->galaxyFits->chisq, vectorLength, PS_TYPE_F32); + for (int k = 0; k < vectorLength; k++) { + outSrc->galaxyFits->Flux->data.F32[k] = sumWeightedFlux->data.F32[k] / sumInvSig2->data.F32[k]; + outSrc->galaxyFits->dFlux->data.F32[k] = 1.0 / sumInvSig2->data.F32[k]; + outSrc->galaxyFits->chisq->data.F32[k] = numerator->data.F32[k] / totalNPix; + +#ifdef nodef + outSrc->galaxyFits->Flux = (psVector *) psBinaryOp(outSrc->galaxyFits->Flux, + sumWeightedFlux, "/", sumInvSig2); + + outSrc->galaxyFits->dFlux = (psVector *) psBinaryOp(outSrc->galaxyFits->dFlux, + psScalarAlloc(1.0, PS_TYPE_F32), "/", sumInvSig2); + + outSrc->galaxyFits->chisq = (psVector *) psBinaryOp(outSrc->galaxyFits->chisq, + numerator, "/", psScalarAlloc(totalNPix, PS_TYPE_F32)); +#endif + } + + psFree(numerator); + psFree(sumInvSig2); + psFree(sumWeightedFlux); +// psFree(tmp); + + int min_j = -1; + psF32 minChisq = NAN; + psVector *chisq = outSrc->galaxyFits->chisq; + for (int j=0; j < chisq->n; j++) { + psF32 thischisq = chisq->data.F32[j]; + if (isfinite(thischisq) && (!isfinite(minChisq) || thischisq < minChisq)) { + min_j = j; + minChisq = thischisq; + } + } + if (min_j >= 0 && isfinite(minChisq) && outSrc->modelEXT) { + // copy the best fit params to the model + psEllipseAxes axes = pmPSF_ModelToAxes(outSrc->modelEXT->params->data.F32, outSrc->modelEXT->type); + axes.major *= options->rMajor->data.F32[min_j]; + axes.minor *= options->rMinor->data.F32[min_j]; + pmPSF_AxesToModel (outSrc->modelEXT->params->data.F32, axes, outSrc->modelEXT->type); + outSrc->modelEXT->chisq = minChisq; + outSrc->modelEXT->mag = -2.5 * log10(outSrc->galaxyFits->Flux->data.F32[min_j]); + outSrc->modelEXT->magErr = 1.0 / + (outSrc->galaxyFits->Flux->data.F32[min_j]/outSrc->galaxyFits->dFlux->data.F32[min_j]); // 1 / SN + } + } + } + + return outSrc; +} + +#define GETVAL(member, key) \ + opt->member = psMetadataLookupF32(&status, md, key); \ + if (!status) { \ + psError (PSPHOT_ERR_CONFIG, true, "failed to looup value for %s in %s", key, \ + useAnalysis ? "readout->analysis" : "recipe"); \ + } + +static bool setOptions(galaxyShapeOptions *opt, pmReadout *readout, psMetadata *recipe, bool makeVectors) { + bool status; + bool useAnalysis; + + psMetadataLookupF32(&useAnalysis, readout->analysis, "GALAXY_SHAPES_FR_MAJOR_MIN"); + psMetadata *md = useAnalysis ? readout->analysis : recipe; + + GETVAL(fRmajorMin, "GALAXY_SHAPES_FR_MAJOR_MIN"); + GETVAL(fRmajorMax, "GALAXY_SHAPES_FR_MAJOR_MAX"); + GETVAL(fRmajorDel, "GALAXY_SHAPES_FR_MAJOR_DEL"); + GETVAL(fRminorMin, "GALAXY_SHAPES_FR_MINOR_MIN"); + GETVAL(fRminorMax, "GALAXY_SHAPES_FR_MINOR_MAX"); + GETVAL(fRminorDel, "GALAXY_SHAPES_FR_MINOR_DEL"); + + opt->numTrials = ceil((opt->fRmajorMax - opt->fRmajorMin + 0.5*opt->fRmajorDel) / opt->fRmajorDel) * + ceil((opt->fRminorMax - opt->fRminorMin + 0.5*opt->fRminorDel) / opt->fRminorDel) ; + + if (makeVectors) { + opt->rMinor = psVectorAlloc(opt->numTrials, PS_TYPE_F32); + opt->rMajor = psVectorAlloc(opt->numTrials, PS_TYPE_F32); + int i = 0; + for (float fRmajor = opt->fRmajorMin; fRmajor < opt->fRmajorMax + 0.5*opt->fRmajorDel; + fRmajor += opt->fRmajorDel) { + for (float fRminor = opt->fRminorMin; fRminor < opt->fRminorMax + 0.5*opt->fRminorDel; + fRminor += opt->fRminorDel) { + opt->rMinor->data.F32[i] = fRminor; + opt->rMajor->data.F32[i] = fRmajor; + i++; + } + } + psAssert(i == opt->numTrials, "Something's wrong with my loop got %d entries expected %d", i, opt->numTrials); + } else { + opt->rMinor = NULL; + opt->rMajor = NULL; + } + + return true; +} + +#define CHECKVAL(left, right, val, message) \ + if (left->val != right.val) { \ + psError (PSPHOT_ERR_CONFIG, true, message); \ + return false; \ + } + +static bool checkOptions(galaxyShapeOptions *options, pmReadout *readout, psMetadata *recipe) { + galaxyShapeOptions thisReadoutsOptions; + + if (!setOptions(&thisReadoutsOptions, readout, recipe, false)) { + psError (PS_ERR_UNKNOWN, false, "problem determining galaxy shape options for readout"); + return false; + } + CHECKVAL(options, thisReadoutsOptions, numTrials, "mismatched number of trials") + CHECKVAL(options, thisReadoutsOptions, fRmajorMin, "mismatched fRmajorMin") + CHECKVAL(options, thisReadoutsOptions, fRmajorMax, "mismatched fRmajorMax") + CHECKVAL(options, thisReadoutsOptions, fRmajorDel, "mismatched fRmajorDel") + CHECKVAL(options, thisReadoutsOptions, fRminorMin, "mismatched fRminorMin") + CHECKVAL(options, thisReadoutsOptions, fRminorMax, "mismatched fRminorMax") + CHECKVAL(options, thisReadoutsOptions, fRminorDel, "mismatched fRminorDel") + + return true; +} Index: branches/eam_branches/ipp-20131211/psphot/src/psphotGalaxyShape.c =================================================================== --- branches/eam_branches/ipp-20131211/psphot/src/psphotGalaxyShape.c (revision 36479) +++ branches/eam_branches/ipp-20131211/psphot/src/psphotGalaxyShape.c (revision 36480) @@ -283,5 +283,13 @@ } +#ifdef SAVE_BEST_MODEL + // Save model with smallest chisq if (isfinite(chisqBest)) { +#else + // Save model with nominal parameters + { + fRmajorBest = 1; + fRminorBest = 1; +#endif // now save the best fitting model as the source's extended model psEllipseAxes testAxes = guessAxes; @@ -293,4 +301,6 @@ psphotGalaxyShapeSource (pcm, source, maskVal, psfSize, false); + // Replace modelEXT with this model + // XXX: only do this if the model is good psFree (source->modelEXT); @@ -299,7 +309,4 @@ source->mode |= PM_SOURCE_MODE_EXTMODEL; source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; - - // adjust the window so the subtraction covers the faint wings - // psphotSetRadiusMoments(&fitRadius, &windowRadius, readout, source, markVal); // cache the model flux Index: branches/eam_branches/ipp-20131211/pstamp/scripts/dqueryparse.pl =================================================================== --- branches/eam_branches/ipp-20131211/pstamp/scripts/dqueryparse.pl (revision 36479) +++ branches/eam_branches/ipp-20131211/pstamp/scripts/dqueryparse.pl (revision 36480) @@ -502,5 +502,5 @@ $command .= " -need_magic" if $need_magic; - my $rlabel = "dq_ud_" . $label if $label; + my $rlabel = "ps_ud_" . $label if $label; $command .= " -rlabel $rlabel" if $rlabel; Index: branches/eam_branches/ipp-20131211/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- branches/eam_branches/ipp-20131211/pstamp/scripts/pstamp_checkdependent.pl (revision 36479) +++ branches/eam_branches/ipp-20131211/pstamp/scripts/pstamp_checkdependent.pl (revision 36480) @@ -27,5 +27,5 @@ my $IPP_DIFF_MODE_STACK_STACK = 4; -my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile); +my ($dep_id, $stage, $stage_id, $component, $imagedb, $label, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile); my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update); @@ -36,5 +36,6 @@ 'component=s' => \$component, 'imagedb=s' => \$imagedb, # dbname for images lookups. - 'rlabel=s' => \$rlabel, + 'label=s' => \$label, # request's label + 'rlabel=s' => \$rlabel, # pstampDependent.rlabel (deprecated) 'need_magic' => \$need_magic, 'fault_count=i' => \$fault_count, @@ -73,4 +74,17 @@ } +if ($label) { + # rlabel is deprecated. Use one based on the supplied label parameter which is the current label + # for the request, which may be different than the one given to the dependent when the job was parsed. + # XXX: having the convention that update label is 'ps_ud_' . $label of request embedded here + # (and formerly in pstampparse.pl) is not particularly clean but it's simple. + my $new_rlabel = 'ps_ud_' . $label; + if ($new_rlabel ne $rlabel) { + print "Notice: using $new_rlabel instead of $rlabel for update label.\n"; + $rlabel = $new_rlabel; + } +} + + if (!$ps_dbserver) { $ps_dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); @@ -559,4 +573,8 @@ } return $warp_status; + } elsif ($warp->{quality} != 0) { + print STDERR "input warp has quality error\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; } # warps are ready fall through and queue the diff update Index: branches/eam_branches/ipp-20131211/tools/mysql-dump/gpc1_install.sh =================================================================== --- branches/eam_branches/ipp-20131211/tools/mysql-dump/gpc1_install.sh (revision 36479) +++ branches/eam_branches/ipp-20131211/tools/mysql-dump/gpc1_install.sh (revision 36480) @@ -62,5 +62,5 @@ log INFO "Symlinking backup file [$TARGET/$BACKUP_FILE] to distribution [$DISTRIBUTION_TARGET]" /usr/bin/ln -f $TARGET/$BACKUP_FILE $DISTRIBUTION_TARGET -/usr/bin/mv -f $TARGET/$MD5FILE $DISTRIBUTION_MD5 +/usr/bin/cp -f $TARGET/$MD5FILE $DISTRIBUTION_MD5 # if it is between 0 and 4, ingest into gpc1_0 or gpc1_1 Index: branches/eam_branches/ipp-20131211/tools/neb_rawOTA_host_scan.pl =================================================================== --- branches/eam_branches/ipp-20131211/tools/neb_rawOTA_host_scan.pl (revision 36479) +++ branches/eam_branches/ipp-20131211/tools/neb_rawOTA_host_scan.pl (revision 36480) @@ -21,9 +21,11 @@ ) or die "Unable to connect to database $DBI::errstr\n"; -my ($host,$min_ins_id,$verbose,$limit,$continue); +my ($host,$min_ins_id,$verbose,$limit,$continue,$alt,$vol_label); $min_ins_id = 0; $verbose = 0; $limit = 10000; $continue = 0; +$alt = 0; +$vol_label = 0; GetOptions( @@ -33,4 +35,6 @@ 'limit|l=s' => \$limit, 'continue|c' => \$continue, + 'alternate|a' => \$alt, + 'vol_label|L=s' => \$vol_label, ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; @@ -39,5 +43,5 @@ defined($host); -my $volume = $host . ".0"; +my $volume = $host . "." . $vol_label; my $rr; @@ -50,6 +54,14 @@ my $vol_id = shift( @{ ${ $r_vol }[0] }); +# Get max ins_id +my $max_id_sth = "SELECT ins_id FROM instance WHERE vol_id = $vol_id ORDER BY ins_id DESC limit 1"; +my $r_max_id = $db->selectall_arrayref( $max_id_sth ); +unless (defined(${ $r_max_id }[0])) { + die "Cannot find max_id"; +} +my $max_ins_id = shift( @{ ${ $r_max_id }[0] }); + if ($verbose) { - print "$host $volume $vol_id $min_ins_id\n"; + print "$host $volume $vol_id $min_ins_id $max_ins_id\n"; } @@ -70,10 +82,13 @@ my $number = 0; do { - my $ins_id_sth = "SELECT ins_id,so_id,uri FROM instance WHERE vol_id = ${vol_id} AND ins_id > $min_ins_id LIMIT $limit"; + my $ins_id_sth = "SELECT ins_id,so_id,uri FROM instance WHERE vol_id = ${vol_id} AND ins_id > $min_ins_id AND ins_id <= $max_ins_id LIMIT $limit"; + if ($alt) { + $ins_id_sth = "SELECT ins_id,so_id,uri,vol_id FROM instance WHERE ins_id > $min_ins_id AND ins_id <= $max_ins_id LIMIT $limit"; + } my $r_ins = $db->selectall_arrayref( $ins_id_sth ); - my $last_ins_id = 0; + my $last_ins_id = $max_ins_id; $number = 0; foreach $rr (@{ $r_ins }) { - my ($ins_id,$so_id,$uri) = @{ $rr }; + my ($ins_id,$so_id,$uri,$v_id) = @{ $rr }; if ($verbose) { # print (" $ins_id $so_id $uri\n"); @@ -81,4 +96,7 @@ $number++; $last_ins_id = $ins_id; + if ($alt) { + if ($v_id != $vol_id) { next; } + } if ($uri !~ /ota...fits/) { next; } my $have_b_node = 0; @@ -138,3 +156,3 @@ $min_ins_id = $last_ins_id; } -} while ($continue && $number != 0); +} while (($continue && $number != 0) || ($min_ins_id < $max_ins_id));