Index: trunk/ippScripts/scripts/camera_exp.pl =================================================================== --- trunk/ippScripts/scripts/camera_exp.pl (revision 19938) +++ trunk/ippScripts/scripts/camera_exp.pl (revision 19942) @@ -19,5 +19,4 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config 1.01 qw( :standard ); @@ -79,40 +78,4 @@ &my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe1; -# values to extract from output metadata and the stats to calculate -# these should be coming from the psastro results -my $CHIPSTATS = - [ - # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG - { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, - { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, - { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, - { name => "bias", type => "mean", flag => "-bias", dtype => "float" }, - { name => "bias_stdev", type => "rms", flag => "-bias_stdev", dtype => "float" }, - { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, - { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, - { name => "fringe_2", type => "stdev", flag => "-fringe_2", dtype => "float" }, - { name => "ap_resid", type => "mean", flag => "-ap_resid", dtype => "float" }, - { name => "ap_resid_stdev", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, - { name => "fwhm_major", type => "mean", flag => "-fwhm_major", dtype => "float" }, - { name => "fwhm_minor", type => "mean", flag => "-fwhm_minor", dtype => "float" }, - { name => "n_stars", type => "sum", flag => "-n_stars", dtype => "int" }, - { name => "n_extended", type => "sum", flag => "-n_extended", dtype => "int" }, - { name => "n_cr", type => "sum", flag => "-n_cr", dtype => "int" }, - ]; -my $chipStats = PS::IPP::Metadata::Stats->new($CHIPSTATS); # Stats parser - -# values to extract from camera-level output metadata and the stats to calculate -my $CAMSTATS = - [ - # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG - { name => "CERROR", type => "rms", flag => "-sigma_ra", dtype => "float" }, - { name => "CERROR", type => "rms", flag => "-sigma_dec", dtype => "float" }, -# { name => "DT_ASTR", type => "sum", flag => "-dtime_astrom", dtype => "float" }, - { name => "NASTRO", type => "sum", flag => "-n_astrom", dtype => "int" }, -# { name => "ZP??", type => "mean", flag => "-zp_mean", dtype => "float" }, -# { name => "ZP??", type => "rms", flag => "-zp_stdev", dtype => "float" }, - ]; -my $camStats = PS::IPP::Metadata::Stats->new($CAMSTATS); # Stats parser - # Look for programs we need my $missing_tools; @@ -120,4 +83,5 @@ my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); # test for addstar and psastro: @@ -131,4 +95,6 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $cmdflags; # Get list of component files @@ -150,7 +116,24 @@ &my_die("Unable to parse metadata list", $cam_id, $PS_EXIT_PROG_ERROR); - # extract the stats from the metadata - unless ($chipStats->parse($metadata)) { - &my_die("Unable to find all values in statistics output.\n", $cam_id, $PS_EXIT_PROG_ERROR); + # since I can't figure out how to do input and output within PERL, I'm writing to a temp file + my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.cm.$cam_id.stats.XXXX", UNLINK => !$save_temps ); + print "saving stats to $statName\n"; + foreach my $line (@$stdout_buf) { + print $statFile $line; + } + close $statFile; + + # parse the stats in the metadata file + $command = "$ppStatsFromMetadata $statName - CAMERA_EXP_IMFILE"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform ppStatsFromMetadata: $error_code\n"); + exit($error_code); + } + + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; } } @@ -248,5 +231,4 @@ if ($chipObjectsExist) { # run psastro on the chipObjects, producing fpaObjects - # XXX add a ppStats call which will collect the astrometry stats my $command; $command = "$psastro -list $list3Name $outroot"; @@ -256,5 +238,5 @@ my $do_stats; if ($run_state eq 'new') { - $command .= " -stats $fpaStats -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS"; $command .= " -dumpconfig $configuration"; $do_stats = 1; @@ -275,23 +257,19 @@ if ($do_stats) { - &my_die("Unable to find expected output file: $fpaStats", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects); - - # Get the statistics on the processed image - my $statsFile; # File handle - open $statsFile, $ipprc->file_resolve($fpaStats) or &my_die("Can't open statistics file $fpaStats: $!", $cam_id, $PS_EXIT_SYS_ERROR); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - - # parse the statistics MDC file - my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files - my $metadata = $mdcParser->parse(join "", @contents); - unless ($metadata) { - &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); - } - - # extract the stats from the metadata - unless ($camStats->parse($metadata)) { - &my_die("Failure extracting metadata from the statistics output file.\n", $cam_id, $PS_EXIT_PROG_ERROR); - } + my $fpaStatsReal = $ipprc->file_resolve($fpaStats); + &my_die("Couldn't find expected output file: $fpaStats", $cam_id, $PS_EXIT_SYS_ERROR) unless -f $fpaStatsReal; + + # parse stats from metadata + $command = "$ppStatsFromMetadata $fpaStatsReal - CAMERA_EXP_FPA"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $cam_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; } @@ -340,5 +318,4 @@ } - my $fpaCommand = "$camtool -cam_id $cam_id"; if ($run_state eq 'new') { @@ -346,6 +323,5 @@ $fpaCommand .= " -uri UNKNOWN"; $fpaCommand .= " -path_base $outroot"; - $fpaCommand .= $chipStats->cmdflags(); - $fpaCommand .= $camStats->cmdflags(); + $fpaCommand .= " $cmdflags"; $fpaCommand .= " -hostname $host" if defined $host; $fpaCommand .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); Index: trunk/ippScripts/scripts/chip_imfile.pl =================================================================== --- trunk/ippScripts/scripts/chip_imfile.pl (revision 19938) +++ trunk/ippScripts/scripts/chip_imfile.pl (revision 19942) @@ -19,5 +19,4 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Config 1.01 qw( :standard ); @@ -74,30 +73,4 @@ } -# values to extract from output metadata and the stats to calculate -# XXX commented-out entries are not yet defined in the output files -my $STATS = - [ - # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG - { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, - { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, - { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, - { name => "OVER_VAL", type => "mean", flag => "-bias", dtype => "float" }, - { name => "OVER_VAL", type => "stdev", flag => "-bias_stdev", dtype => "float" }, - { name => "FRINGE_0", type => "rms", flag => "-fringe_0", dtype => "float" }, - { name => "FRINGE_RESID_0", type => "rms", flag => "-fringe_1", dtype => "float" }, - { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_2", dtype => "float" }, - { name => "APMIFIT", type => "mean", flag => "-ap_resid", dtype => "float" }, - { name => "DAPMIFIT", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, - { name => "FWHM_X", type => "mean", flag => "-fwhm_major", dtype => "float" }, - { name => "FWHM_Y", type => "mean", flag => "-fwhm_minor", dtype => "float" }, - { name => "DT_DET", type => "sum", flag => "-dtime_detrend", dtype => "float" }, - { name => "DT_PHOT", type => "sum", flag => "-dtime_photom", dtype => "float" }, - { name => "DT_TOTAL", type => "sum", flag => "-dtime_total", dtype => "float" }, - { name => "NSTARS", type => "sum", flag => "-n_stars", dtype => "int" }, -# { name => "?", type => "sum", flag => "-n_extended", dtype => "int" }, -# { name => "?", type => "sum", flag => "-n_cr", dtype => "int" }, - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for programs we need my $missing_tools; @@ -105,4 +78,5 @@ my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -137,8 +111,8 @@ } +my $cmdflags; + # Run ppImage unless ($no_op) { - ## XXX can we convert ppImage log and trace to use the filerules to generate consistent names - ## XXX also stats: output should be implied by $outroot my $command; my $do_stats; @@ -205,21 +179,19 @@ &my_die("Couldn't find expected output file: $outputStats\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); - # Get the statistics on the processed image - my $statsFile; # File handle - open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - - # parse the statistics MDC file - my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files - my $metadata = $mdcParser->parse(join "", @contents); - unless ($metadata) { - &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); - } - - # extract the stats from the metadata - unless ($stats->parse($metadata)) { - &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); - } + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal; + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - CHIP_IMFILE"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $exp_id, $chip_id, $class_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; } } @@ -236,5 +208,5 @@ $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; - $command .= $stats->cmdflags(); + $command .= " $cmdflags"; $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); } else { Index: trunk/ippScripts/scripts/detrend_norm_apply.pl =================================================================== --- trunk/ippScripts/scripts/detrend_norm_apply.pl (revision 19938) +++ trunk/ippScripts/scripts/detrend_norm_apply.pl (revision 19942) @@ -109,5 +109,5 @@ my $outFile = ($det_type_real eq "mask") ? 'PPIMAGE.OUTPUT.DETMASK' : 'PPIMAGE.OUTPUT';; # XXXX something of a hack (too many places to control things...) -my $RECIPE_PPSTATS = ($det_type_real eq "dark") ? 'DARKSTATS' : 'CHIPSTATS';; # XXXX something of a hack (too many places to control things...) +my $RECIPE_PPSTATS = ($det_type_real eq "dark") ? 'DARKSTATS' : 'DETSTATS';; # XXXX something of a hack (too many places to control things...) my $output = $ipprc->filename($outFile, $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); @@ -153,6 +153,6 @@ &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $statsNameReal; - # ppStatsFromMetadata $outputStats - DETREND_RESID_IMFILE - $command = "$ppStatsFromMetadata $statsNameReal - DETREND_RESID_IMFILE"; + # parse stats from metadata + $command = "$ppStatsFromMetadata $statsNameReal - DETREND_NORM_APPLY"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: trunk/ippScripts/scripts/detrend_process_exp.pl =================================================================== --- trunk/ippScripts/scripts/detrend_process_exp.pl (revision 19938) +++ trunk/ippScripts/scripts/detrend_process_exp.pl (revision 19942) @@ -16,5 +16,4 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Config 1.01 qw( :standard ); use PS::IPP::Metadata::List qw( parse_md_list ); @@ -68,26 +67,15 @@ &my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe; -# values to extract from output metadata and the stats to calculate -# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) -# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) -# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 -my $STATS = - [ - # KEYWORD STATISTIC CHIPTOOL FLAG - { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, - { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, - { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, - # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for programs we need my $missing_tools; my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); exit($PS_EXIT_CONFIG_ERROR); } + +my $cmdflags; # Get list of component files @@ -112,7 +100,24 @@ &my_die("Unable to parse metadata list", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); - # parse the stats in the metadata - unless ($stats->parse($metadata)) { - &my_die("Unable to find all values in statistics output.\n", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + # since I can't figure out how to do input and output within PERL, I'm writing to a temp file + my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detproc.$det_id.stats.XXXX", UNLINK => !$save_temps ); + print "saving stats to $statName\n"; + foreach my $line (@$stdout_buf) { + print $statFile $line; + } + close $statFile; + + # parse the stats in the metadata file + $command = "$ppStatsFromMetadata $statName - DETREND_PROCESS_EXP"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform ppStatsFromMetadata: $error_code\n"); + exit($error_code); + } + + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; } } @@ -173,5 +178,5 @@ $command .= " -recip $recipe -path_base $outroot"; $command .= " -dbname $dbname" if defined $dbname; -$command .= $stats->cmdflags(); +$command .= " $cmdflags"; # Add the processed file to the database Index: trunk/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- trunk/ippScripts/scripts/detrend_process_imfile.pl (revision 19938) +++ trunk/ippScripts/scripts/detrend_process_imfile.pl (revision 19942) @@ -16,5 +16,4 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Config 1.01 qw( :standard ); @@ -68,18 +67,9 @@ my $jpeg_recipe = $ipprc->reduction($reduction, uc($det_type) . '_JPEG_IMAGE'); # Recipe name for JPEG -# values to extract from output metadata and the stats to calculate -my $STATS = - [ - # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG - { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, - { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, - { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for programs we need my $missing_tools; my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -97,4 +87,6 @@ $ipprc->outroot_prepare($outroot); +my $cmdflags; + my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); @@ -107,5 +99,5 @@ $command .= " -recipe PPIMAGE $ppimage_recipe"; $command .= " -recipe JPEG $jpeg_recipe"; - $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -recipe PPSTATS DETSTATS"; $command .= " -stats $outputStats"; $command .= " -tracedest $traceDest -log $logDest"; @@ -121,21 +113,22 @@ &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); - &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); &my_die("Couldn't find expected output file: $outputBin1", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin1); &my_die("Couldn't find expected output file: $outputBin2", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin2); - # Get the statistics on the processed image - my $statsFile; # File handle - open $statsFile, $ipprc->file_resolve("$outputStats") or die "Can't open statistics file $outputStats: $!\n"; - my @contents = <$statsFile>; # Contents of file - close $statsFile; + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal; - # parse the statistics MDC file - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - my $metadata = $mdcParser->parse(join "", @contents) - or &my_die("Unable to parse metadata config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); - - # extract the stats from the metadata - $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + # parse stats from metadata + $command = "$ppStatsFromMetadata $outputStatsReal - DETREND_PROCESS_IMFILE"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; } @@ -148,5 +141,5 @@ $command .= " -uri $outputImage -path_base $outroot"; $command .= " -dbname $dbname" if defined $dbname; -$command .= $stats->cmdflags(); +$command .= " $cmdflags"; # Add the processed file to the database Index: trunk/ippScripts/scripts/detrend_resid_exp.pl =================================================================== --- trunk/ippScripts/scripts/detrend_resid_exp.pl (revision 19938) +++ trunk/ippScripts/scripts/detrend_resid_exp.pl (revision 19942) @@ -128,4 +128,5 @@ close $statFile; + # parse the stats in the metadata file $command = "$ppStatsFromMetadata $statName - DETREND_RESID_EXP"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = Index: trunk/ippScripts/scripts/detrend_stack.pl =================================================================== --- trunk/ippScripts/scripts/detrend_stack.pl (revision 19938) +++ trunk/ippScripts/scripts/detrend_stack.pl (revision 19942) @@ -16,5 +16,4 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config 1.01 qw( :standard ); @@ -67,21 +66,9 @@ my $recipe = $ipprc->reduction($reduction, $det_type . '_STACK'); # Recipe name to use -# values to extract from output metadata and the stats to calculate -# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) -# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) -# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 -my $STATS = - [ - # KEYWORD STATISTIC DETTOOL FLAG - { name => "ROBUST_MEDIAN", type => "clipmean", flag => "-bg", dtype => "float" }, - { name => "ROBUST_MEDIAN", type => "clipstdev", flag => "-bg_mean_stdev", dtype => "float" }, - { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for programs we need my $missing_tools; my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); my $ppMerge = can_run('ppMerge') or (warn "Can't find ppMerge" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -112,23 +99,25 @@ # The stats recipe depends on the detrend type -my $STATRECIPES = {'FLATMASK' => 'CHIPSTATS', - 'DARKMASK' => 'CHIPSTATS', - 'MASK' => 'CHIPSTATS', - 'BIAS' => 'CHIPSTATS', +my $STATRECIPES = {'FLATMASK' => 'DETSTATS', + 'DARKMASK' => 'DETSTATS', + 'MASK' => 'DETSTATS', + 'BIAS' => 'DETSTATS', 'DARK' => 'DARKSTATS', 'DARK_PREMASK' => 'DARKSTATS', 'SHUTTER' => 'DARKSTATS', - 'FLAT_PREMASK' => 'CHIPSTATS', - 'DOMEFLAT_PREMASK' => 'CHIPSTATS', - 'SKYFLAT_PREMASK' => 'CHIPSTATS', - 'FLAT_RAW' => 'CHIPSTATS', - 'DOMEFLAT_RAW' => 'CHIPSTATS', - 'SKYFLAT_RAW' => 'CHIPSTATS', - 'FLAT' => 'CHIPSTATS', - 'DOMEFLAT' => 'CHIPSTATS', - 'SKYFLAT' => 'CHIPSTATS', - 'FRINGE' => 'CHIPSTATS', + 'FLAT_PREMASK' => 'DETSTATS', + 'DOMEFLAT_PREMASK' => 'DETSTATS', + 'SKYFLAT_PREMASK' => 'DETSTATS', + 'FLAT_RAW' => 'DETSTATS', + 'DOMEFLAT_RAW' => 'DETSTATS', + 'SKYFLAT_RAW' => 'DETSTATS', + 'FLAT' => 'DETSTATS', + 'DOMEFLAT' => 'DETSTATS', + 'SKYFLAT' => 'DETSTATS', + 'FRINGE' => 'DETSTATS', }; my $statrecipe = $STATRECIPES->{$det_type}; # File rule for output + +my $cmdflags; # Get list of files to stack @@ -217,18 +206,20 @@ &my_die("Unable to find expected output file: $outputCount\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputCount); &my_die("Unable to find expected output file: $outputSigma\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputSigma); - &my_die("Unable to find expected output file: $outputStats\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); - - # Get the statistics on the stacked image - open(my $statsFile, $ipprc->file_resolve("$outputStats")) or - &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); - my $contents = do { local $/; <$statsFile> }; # Contents of file - close($statsFile); - - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - my $metadata = $mdcParser->parse($contents) or - &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); - - $stats->parse($metadata) or - &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal; + + # parse stats from metadata + $command = "$ppStatsFromMetadata $outputStatsReal - DETREND_STACK"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $det_id, $iter, $class_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; } @@ -241,5 +232,5 @@ $command .= " -recip $recipe"; $command .= " -dbname $dbname" if defined $dbname; -$command .= $stats->cmdflags(); +$command .= " $cmdflags"; # Add the resultant into the database