Index: trunk/ippScripts/scripts/fake_imfile.pl
===================================================================
--- trunk/ippScripts/scripts/fake_imfile.pl	(revision 17631)
+++ trunk/ippScripts/scripts/fake_imfile.pl	(revision 17631)
@@ -0,0 +1,274 @@
+#!/usr/bin/env perl
+
+## this script is run on every imfile to perform the fake source
+## analysis and the forced photometry analysis.  
+
+## For the fake source analysis, we load the image, inject a number of
+## fake source (with a flat mag distribution), save the input source,
+## then perform photometry on the image to recover the input sources.
+## we should save a file with the matched input and recovered
+## sources.  the output metadata should include an analysis of the
+## recovery rate and measurement error as a function of magnitude
+
+## For the forced photometry, we need to use the measured astrometry
+## to select the desired locations in pixel coord.  
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::Stats;
+
+use PS::IPP::Config qw($PS_EXIT_SUCCESS
+		       $PS_EXIT_UNKNOWN_ERROR
+		       $PS_EXIT_SYS_ERROR
+		       $PS_EXIT_CONFIG_ERROR
+		       $PS_EXIT_PROG_ERROR
+		       $PS_EXIT_DATA_ERROR
+		       $PS_EXIT_TIMEOUT_ERROR
+		       metadataLookupStr
+		       metadataLookupBool
+		       caturi
+		       );
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Parse the command-line arguments
+my ( $exp_id, $exp_tag, $chip_id, $class_id, $uri, $camera, $outroot, $dbname, $reduction, $verbose,
+     $no_update, $no_op );
+GetOptions(
+	   'exp_id=s'      => \$exp_id,    # Exposure identifier
+	   'exp_tag=s'     => \$exp_tag,   # Exposure identifier
+	   'chip_id=s'     => \$chip_id,   # Chiptool identifier
+	   'class_id=s'    => \$class_id,  # Class identifier
+	   'uri|u=s'       => \$uri,       # Input FITS file
+	   'camera|c=s'    => \$camera,	   # Camera
+	   'outroot|w=s'   => \$outroot,   # output file base name
+	   'dbname|d=s'    => \$dbname,    # Database name
+	   'reduction=s'   => \$reduction, # Reduction class
+	   'verbose'       => \$verbose,   # Print to stdout
+	   'no-update'     => \$no_update, # Don't update the database?
+	   'no-op'         => \$no_op,	   # Don't do any operations?
+	   ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --exp_id --exp_tag --chip_id --class_id --uri --camera --outroot",
+	   -exitval => 3) unless
+    defined $exp_id and
+    defined $exp_tag and
+    defined $chip_id and
+    defined $class_id and
+    defined $uri and
+    defined $camera and
+    defined $outroot;
+
+$ipprc->define_camera($camera);
+
+# Recipes to use based on reduction class
+# the CHIP recipe should not perform astrometry anymore (2008.04.08)
+
+$reduction = 'DEFAULT' unless defined $reduction;
+my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use
+unless ($recipe) {
+    &my_die("Couldn't find selected reduction for CHIP: $reduction\n", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+}
+
+# values to extract from output metadata and the stats to calculate
+# XXX commented-out entries are not yet defined in the output files
+# XXX EAM : we are removing the astrometry analysis from chip, so we
+# do not bother to calculate those stats.
+my $STATS =
+   [
+       #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
+       { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg",             dtype => "float" },
+       { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev",  dtype => "float" },
+       { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev",       dtype => "float" },
+       { name => "OVER_VAL",       type => "mean",  flag => "-bias", 	       dtype => "float" },
+       { name => "OVER_VAL",       type => "stdev", flag => "-bias_stdev",     dtype => "float" },
+       { name => "FRINGE_0",       type => "rms",   flag => "-fringe_0",       dtype => "float" },
+       { name => "FRINGE_RESID_0", type => "rms",   flag => "-fringe_1",       dtype => "float" },
+       { name => "FRINGE_ERR_0",   type => "rms",   flag => "-fringe_2",       dtype => "float" },
+#      { name => "CERROR",         type => "rms",   flag => "-sigma_ra",       dtype => "float" },
+#      { name => "CERROR",         type => "rms",   flag => "-sigma_dec",      dtype => "float" },
+       { name => "APMIFIT",        type => "mean",  flag => "-ap_resid",       dtype => "float" },
+       { name => "DAPMIFIT",       type => "rms",   flag => "-ap_resid_stdev", dtype => "float" },
+#      { name => "ZP??",           type => "mean",  flag => "-zp_mean",        dtype => "float" },
+#      { name => "ZP??",           type => "rms",   flag => "-zp_stdev",       dtype => "float" },
+       { name => "FWHM_X",         type => "mean",  flag => "-fwhm_major",     dtype => "float" },
+       { name => "FWHM_Y",         type => "mean",  flag => "-fwhm_minor",     dtype => "float" },
+       { name => "DT_DET",         type => "sum",   flag => "-dtime_detrend",  dtype => "float" },
+       { name => "DT_PHOT",        type => "sum",   flag => "-dtime_photom",   dtype => "float" },
+#       { name => "DT_ASTR",        type => "sum",   flag => "-dtime_astrom",   dtype => "float" },
+       { name => "NSTARS",         type => "sum",   flag => "-n_stars",        dtype => "int"   },
+#      { name => "?",              type => "sum",   flag => "-n_extended",     dtype => "int"   },
+#      { name => "?",              type => "sum",   flag => "-n_cr",           dtype => "int"   },
+#      { name => "NASTRO",         type => "sum",   flag => "-n_astrom",       dtype => "int"   },
+   ];
+my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+
+# Look for programs we need
+my $missing_tools;
+my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
+
+&my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
+
+# outroot examples (HOST components must be set)
+# file://data/ipp004.0/gpc1/20080130
+# neb:///ipp004-v1/gpc1/20080130
+# neb:///*/gpc1/20080130 (volume not specified)
+
+# check for existing directory, generate if needed
+$ipprc->outroot_prepare($outroot);
+
+## these names are used in ppImage, and thus may be URIs
+my $outputImage   = $ipprc->filename("PPIMAGE.CHIP",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $outputMask    = $ipprc->filename("PPIMAGE.CHIP.MASK",   $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $outputWeight  = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $outputBin1    = $ipprc->filename("PPIMAGE.BIN1",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $outputBin2    = $ipprc->filename("PPIMAGE.BIN2",  	    $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $outputStats   = $ipprc->filename("PPIMAGE.STATS", 	    $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $traceDest     = $ipprc->filename("TRACE.IMFILE", 	    $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $logDest       = $ipprc->filename("LOG.IMFILE", 	    $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+
+# Run ppImage
+unless ($no_op) {
+    ## XXX can we convert ppImage log and trace to use the filerules to generate consistent names
+    ## XXX also stats: output should be implied by $outroot
+    my $command = "$ppImage -file $uri $outroot";
+    $command .= " -recipe PPIMAGE $recipe";
+    $command .= " -recipe PPSTATS CHIPSTATS";
+    $command .= " -stats $outputStats";
+    $command .= " -dbname $dbname" if defined $dbname;
+    $command .= " -tracedest $traceDest -log $logDest";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform ppImage: $error_code", $exp_id, $chip_id, $class_id, $error_code);
+    }
+
+    ## get the ppImage recipe for this camera and CHIP reduction
+    $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe -";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+
+    ## allow the output images to be optional, depending on the recipe / reduction class
+    my $outputImageExpect = metadataLookupBool($recipeData, 'CHIP.FITS');
+    if ($outputImageExpect) {
+	&my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
+    }
+
+    my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS');
+    if ($outputMaskExpect) {
+	&my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
+    }
+
+    my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.WEIGHT.FITS');
+    if ($outputWeightExpect) {
+	&my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
+    }
+
+    &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
+    &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
+    &my_die("Couldn't find expected output file: $outputStats\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats);
+
+    # Get the statistics on the processed image
+    my $statsFile;		# File handle
+    open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    my @contents = <$statsFile>; # Contents of file
+    close $statsFile;
+
+    # parse the statistics MDC file
+    my $mdcParser = PS::IPP::Metadata::Config->new();	# Parser for metadata config files
+    my $metadata = $mdcParser->parse(join "", @contents);
+    unless ($metadata) {
+	&my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
+    }
+
+    # extract the stats from the metadata
+    unless ($stats->parse($metadata)) {
+	&my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
+    }
+}
+
+# command to update database
+my $command = "$chiptool -addprocessedimfile";
+$command .= " -exp_id $exp_id";
+$command .= " -chip_id $chip_id";
+$command .= " -class_id $class_id";
+$command .= " -uri $outputImage";
+$command .= " -path_base $outroot";
+$command .= " -dbname $dbname" if defined $dbname;
+$command .= $stats->cmdflags();
+
+# Add the processed file to the database
+unless ($no_update) {
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	warn("Unable to perform chiptool -addprocessedimfile: $error_code\n");
+	exit($error_code);
+    }
+} else {
+    print "skipping command: $command\n";
+}
+
+sub my_die
+{
+    my $msg = shift; # Warning message on die
+    my $exp_id = shift; # Chiptool identifier
+    my $chip_id = shift; # Chiptool identifier
+    my $class_id = shift; # Class identifier
+    my $exit_code = shift; # Exit code to add
+
+    carp($msg);
+    if (defined $chip_id and defined $class_id and not $no_update) {
+	my $command = "$chiptool -addprocessedimfile";
+	$command .= " -exp_id $exp_id";
+	$command .= " -chip_id $chip_id";
+	$command .= " -class_id $class_id";
+	$command .= " -code $exit_code";
+	$command .= " -uri $outputImage";
+	$command .= " -dbname $dbname" if defined $dbname;
+        system ($command);
+    }
+    exit $exit_code;
+}
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+__END__