Index: /branches/eam_branches/ipp-20230313/ippScripts/Build.PL =================================================================== --- /branches/eam_branches/ipp-20230313/ippScripts/Build.PL (revision 42601) +++ /branches/eam_branches/ipp-20230313/ippScripts/Build.PL (revision 42602) @@ -159,4 +159,5 @@ scripts/xcsky.pl scripts/xccal.pl + scripts/xcff_warp.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /branches/eam_branches/ipp-20230313/ippScripts/scripts/xccal.pl =================================================================== --- /branches/eam_branches/ipp-20230313/ippScripts/scripts/xccal.pl (revision 42601) +++ /branches/eam_branches/ipp-20230313/ippScripts/scripts/xccal.pl (revision 42602) @@ -45,4 +45,5 @@ print "USAGE: xccal.pl --xccal_id (xccal_id) --outroot (root) --camera (camera) [options]\n"; print " --xccal_id (xccal_id) : run identifier\n"; + print " --xcstack_id (xcstack_id) : run identifier\n"; print " --camera (camera) : Camera name\n"; print " --dbname (dbname) : Database name\n"; Index: /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcff_warp.pl =================================================================== --- /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcff_warp.pl (revision 42602) +++ /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcff_warp.pl (revision 42602) @@ -0,0 +1,331 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Scalar::Util qw(looks_like_number); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $psphotFullForce = can_run('psphotFullForce') or (warn "Can't find psphotFullForce" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $xcfftool = can_run('xcfftool') or (warn "Can't find xcfftool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($xcff_id, $warp_id, $skycell_id, $path_base, $sourceroot, $camera); +my ($outroot, $reduction); +my ($dbname, $threads, $verbose, $no_update, $no_op, $redirect); + +sub HelpMessage { + print "USAGE: xcff_warp.pl --xcff_id (xccal_id) --outroot (root) --camera (camera) [options]\n"; + print " --xcff_id (xccal_id) : run identifier\n"; + print " --warp_id (warp_id) : warp identifier\n"; + print " --skycell_id (skycell_id) : skycell identifier\n"; + print " --warp_path_base (path_base) : path_base of input\n"; + print " --sources_path_base (path_base) : path_base of input\n"; + print " --camera (camera) : Camera name\n"; + print " --dbname (dbname) : Database name\n"; + print " --threads (N) : Number of threads to use\n"; + print " --outroot (root) : Output root name\n"; + print " --reduction (class) : Reduction class\n"; + print " --verbose : Be extra verbose\n"; + print " --no-update : Don't update the database?\n"; + print " --no-op : Don't do any operations?\n"; + print " --redirect-output : send output to the log file,\n"; + print " --help : show this message\n"; + exit 2; +} + +GetOptions( + 'xcff_id=s' => \$xcff_id, + 'warp_id=s' => \$warp_id, # warp identifier + 'skycell_id=s' => \$skycell_id,# Skycell identifier + 'warp_path_base=s' => \$path_base, # path_base of the warp skycell --- could get this from the db + 'sources_path_base=s' => \$sourceroot,# path_base of sources --- could get this from the db + 'camera=s' => \$camera, # camera name of sources + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'help' => sub { HelpMessage() }, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --xcff_id --warp_id --sources_path_base --skycell_id --warp_path_base --outroot --camera", -exitval => 3, + ) unless defined $xcff_id + and defined $warp_id + and defined $sourceroot + and defined $skycell_id + and defined $path_base + and defined $camera + and defined $outroot; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { $neb = $ipprc->nebulous(); } + +my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id); + +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'FULLFORCE_PSPHOT'); # Recipe to use for psphot +unless ($recipe_psphot) { &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR); } + +my $doStats = 1; + +print "reduction: $reduction\n"; +print "recipe_psphot: $recipe_psphot\n"; + +# XXX: get this from recipe +# use psf measured on input warp (if false, PSF is generated by psphotFullForce?) +my $useWarpPSF = 0; + +my $input = $ipprc->filename('PSWARP.OUTPUT', $path_base); +my $inputMask = $ipprc->filename('PSWARP.OUTPUT.MASK', $path_base); +my $inputVariance = $ipprc->filename('PSWARP.OUTPUT.VARIANCE', $path_base); +my $inputPSF = $useWarpPSF ? $ipprc->filename('PSPHOT.PSF.SKY.SAVE', $path_base) : ""; +my $inputSources = $ipprc->filename('PSPHOT.OUTPUT.CFF', $sourceroot); + +if ($verbose) { + print "input: $input\n"; + print "inputMask: $inputMask\n"; + print "inputVariance: $inputVariance\n"; + print "inputPSF: $inputPSF\n" if $inputPSF; + print "inputSources: $inputSources\n"; +} + +# check that the inputs exist (and have non-zero size) +&my_die("Couldn't find input: $input", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); +&my_die("Couldn't find input: $inputMask", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputVariance", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance); +&my_die("Couldn't find input: $inputPSF", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) if ($inputPSF && !$ipprc->file_exists($inputPSF)); +&my_die("Couldn't find input: $inputSources", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputSources); + +my $dump_config = 1; + +# Get the output filenames +my $outputSources = prepare_output("PSPHOT.OUT.CMF.MEF", $outroot, 1); +my $configuration = prepare_output("PSPHOT.SKY.CONFIG", $outroot, 1) if $dump_config; +my $outputStats = prepare_output("SKYCELL.STATS", $outroot, 1) if $doStats; +my $traceDest = prepare_output("TRACE.EXP", $outroot, 1); + +my $cmdflags = ""; +my $quality = 0; + +# Perform psphotFullForce +{ + my $command = "$psphotFullForce $outroot"; + $command .= " -force $inputSources"; + $command .= " -file $input"; + $command .= " -mask $inputMask"; + $command .= " -variance $inputVariance"; + $command .= " -psf $inputPSF" if $inputPSF; + $command .= " -threads $threads" if defined $threads; + if ($dump_config) { + $command .= " -dumpconfig $configuration"; + } + $command .= " -recipe PSPHOT $recipe_psphot"; + if ($doStats) { + $command .= " -stats $outputStats"; + } + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + $command .= " -photcode-rule '{DETECTOR}.{FILTER.ID}.ForcedWarp'"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $warp_id, $skycell_id, $error_code); + } + + if ($doStats) { + check_output($outputStats, 1); + my $outputStatsReal = $ipprc->file_resolve($outputStats); + + # analyze the stats file + $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $warp_id, $skycell_id, $error_code); + } + foreach my $line (@$stdout_buf) { $cmdflags .= " $line"; } + chomp $cmdflags; + } + ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + } else { + print "Not executing: $command\n"; + } +} + +if (!$quality) { + check_output($outputSources, 1); +} + +# Add the result to the database +{ + my $command = "$xcfftool -xcff_id $xcff_id -warp_id $warp_id"; + $command .= " -addresult -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $warp_id, $skycell_id, $error_code); + } + } else { + print "Not executing $command\n"; + } +} + +exit 0; + + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $xcff_id, $warp_id, $skycell_id, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $xcff_id = shift; # full force run identifier + my $warp_id = shift; # full force warp id + my $skycell_id = shift; # Skycell identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $xcff_id and defined $skycell_id) { + my $command = "$xcfftool -xcff_id $xcff_id -warp_id $warp_id -fault $exit_code"; + $command .= " -addresult"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + run(command => $command, verbose => $verbose); + } else { + print "not executing $command\n"; + } + } + exit $exit_code; +} + +# if the program exited normally, the exit value is returned +# if the program exited with a signal (abort, crash, etc), PS_EXIT_PROG_ERROR (4) is returned +# if the program failed to run, -PS_EXIT_PROG_ERROR (-4) is returned +sub parse_error_message { + my $success = shift; + my $error_msg = shift; + my $command = shift; + + if ($success) { return 0; } + + print "raw error_msg: $error_msg\n"; + print "full command: $command\n"; + + if (&looks_like_number($error_msg)) { return (($error_msg >> 8) or ($PS_EXIT_PROG_ERROR)); } + + # which of these match? + my ($error_code) = $error_msg =~ m/exited with value (\d+)/; + if (defined $error_code) { return $error_code; } + + ($error_code) = $error_msg =~ m/died with signal (\d+)/; +# if (defined $error_code) { return (128 + $error_code); } + if (defined $error_code) { return ($PS_EXIT_PROG_ERROR); } + + # probably failed to execute: + return (-1*$PS_EXIT_PROG_ERROR); +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcsky.pl =================================================================== --- /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcsky.pl (revision 42601) +++ /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcsky.pl (revision 42602) @@ -135,15 +135,15 @@ { - my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT'); # Recipe to use for psphot - my $recipe_ppsub = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB'); # Recipe to use for ppsub - my $recipe_ppstack = $ipprc->reduction($reduction, 'STACKPHOT_PPSTACK'); # Recipe to use for ppstack + my $recipe_psphot = $ipprc->reduction($reduction, 'XCSTACK_PSPHOT'); # Recipe to use for psphot + # my $recipe_ppsub = $ipprc->reduction($reduction, 'XCSTACK_PPSUB'); # Recipe to use for ppsub + # my $recipe_ppstack = $ipprc->reduction($reduction, 'XCSTACK_PPSTACK'); # Recipe to use for ppstack unless ($recipe_psphot) { - &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $xcsky_id, $PS_EXIT_CONFIG_ERROR); + &my_die("Couldn't find selected reduction for XCSTACK_PSPHOT: $reduction\n", $xcsky_id, $PS_EXIT_CONFIG_ERROR); } print "reduction: $reduction\n"; print "recipe_psphot: $recipe_psphot\n"; - print "recipe_ppsub: $recipe_ppsub\n"; - print "recipe_ppstack: $recipe_ppstack\n"; + # print "recipe_ppsub: $recipe_ppsub\n"; + # print "recipe_ppstack: $recipe_ppstack\n"; my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); @@ -261,6 +261,6 @@ $command .= " -threads $threads" if defined $threads; $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -recipe PPSUB $recipe_ppsub"; - $command .= " -recipe PPSTACK $recipe_ppstack"; + # $command .= " -recipe PPSUB $recipe_ppsub"; + # $command .= " -recipe PPSTACK $recipe_ppstack"; $command .= " -dumpconfig $configuration" if !$updatemode; $command .= " -tracedest $traceDest -log $logDest"; @@ -274,4 +274,5 @@ } + # XXX need to enable the STATS output in psphotStack, then read the results here: # my $outputStatsReal = $ipprc->file_resolve($outputStats); # &my_die("Couldn't find expected output file: $outputStats", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); Index: /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcstack_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcstack_skycell.pl (revision 42601) +++ /branches/eam_branches/ipp-20230313/ippScripts/scripts/xcstack_skycell.pl (revision 42602) @@ -19,4 +19,5 @@ use IPC::Cmd 0.36 qw( can_run run ); +use Scalar::Util qw(looks_like_number); use PS::IPP::Metadata::Config; use PS::IPP::Metadata::List qw( parse_md_list ); @@ -244,9 +245,8 @@ # cannot use run_command here (no metadata output to parse) - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppStack: $error_code", $xcstack_id, $error_code); + &my_die("Unable to perform $command: $error_msg", $xcstack_id, $error_code); } @@ -259,8 +259,8 @@ # measure stats $command = "$ppStatsFromMetadata $outputStatsReal - XCSTACK_SKYCELL"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppStatsFromMetadata: $error_code", $xcstack_id, $error_code); + &my_die("Unable to perform $command: $error_msg", $xcstack_id, $error_code); } foreach my $line (@$stdout_buf) { $cmdflags .= " $line"; } @@ -295,16 +295,13 @@ $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform xcstacktool $mode $error_code", $xcstack_id, $error_code); + &my_die("Unable to perform $command: $error_msg", $xcstack_id, $error_code); } } - } print "I've updated the database: $xcstack_id\n"; - print "I've reached the end of processing with a code of $?: $xcstack_id\n"; @@ -411,8 +408,8 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command", $xcstack_id, $error_code); + &my_die("Unable to perform $command: $error_msg", $xcstack_id, $error_code); } @@ -435,4 +432,31 @@ } +# if the program exited normally, the exit value is returned +# if the program exited with a signal (abort, crash, etc), PS_EXIT_PROG_ERROR (4) is returned +# if the program failed to run, -PS_EXIT_PROG_ERROR (-4) is returned +sub parse_error_message { + my $success = shift; + my $error_msg = shift; + my $command = shift; + + if ($success) { return 0; } + + print "raw error_msg: $error_msg\n"; + print "full command: $command\n"; + + if (&looks_like_number($error_msg)) { return (($error_msg >> 8) or ($PS_EXIT_PROG_ERROR)); } + + # which of these match? + my ($error_code) = $error_msg =~ m/exited with value (\d+)/; + if (defined $error_code) { return $error_code; } + + ($error_code) = $error_msg =~ m/died with signal (\d+)/; +# if (defined $error_code) { return (128 + $error_code); } + if (defined $error_code) { return ($PS_EXIT_PROG_ERROR); } + + # probably failed to execute: + return (-1*$PS_EXIT_PROG_ERROR); +} + __END__