Index: /trunk/ippScripts/scripts/diff_skycell.pl =================================================================== --- /trunk/ippScripts/scripts/diff_skycell.pl (revision 20100) +++ /trunk/ippScripts/scripts/diff_skycell.pl (revision 20101) @@ -18,5 +18,4 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Metadata::List qw( parse_md_list ); use Data::Dumper; @@ -53,31 +52,9 @@ $ipprc->redirect_output($logDest) if $redirect; -my $STATS = - [ - # PPSTATS KEYWORD STATISTIC DIFFTOOL FLAG - { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, - { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, - { name => "TIME_SUB", type => "sum", flag => "-dtime_diff", dtype => "float" }, - { name => "TIME_MATCH", type => "sum", flag => "-dtime_match", dtype => "float" }, - { name => "TIME_PHOT", type => "sum", flag => "-dtime_phot", dtype => "float" }, - { name => "SUBTRACTION.STAMPS", type => "mean", flag => "-stamps_num", dtype => "int" }, - { name => "SUBTRACTION.DEV.MEAN", type => "mean", flag => "-stamps_mean", dtype => "float" }, - { name => "SUBTRACTION.DEV.RMS", type => "mean", flag => "-stamps_rms", dtype => "float" }, - { name => "SUBTRACTION.NORM", type => "mean", flag => "-norm", dtype => "float" }, - { name => "SUBTRACTION.BGDIFF", type => "mean", flag => "-bg_diff", dtype => "float" }, - { name => "SUBTRACTION.MX", type => "mean", flag => "-kernel_x", dtype => "float" }, - { name => "SUBTRACTION.MY", type => "mean", flag => "-kernel_y", dtype => "float" }, - { name => "SUBTRACTION.MXX", type => "mean", flag => "-kernel_xx", dtype => "float" }, - { name => "SUBTRACTION.MXY", type => "mean", flag => "-kernel_xy", dtype => "float" }, - { name => "SUBTRACTION.MYY", type => "mean", flag => "-kernel_yy", dtype => "float" }, - { name => "NUM_SOURCES", type => "sum", flag => "-sources", dtype => "int" }, - { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for programs we need my $missing_tools; my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -204,4 +181,6 @@ my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $cmdflags; + # Perform subtraction unless ($no_op) { @@ -233,14 +212,20 @@ # &my_die("Couldn't find expected output file: $bin1Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); # &my_die("Couldn't find expected output file: $bin2Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); - &my_die("Couldn't find expected output file: $outputStats", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); - - # Get the statistics on the residual image - my $statsFile; # File handle - open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $diff_id, $PS_EXIT_SYS_ERROR); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - my $metadata = $mdcParser->parse(join "", @contents) or - &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); - $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $diff_id, $PS_EXIT_PROG_ERROR); + + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $diff_id, $PS_EXIT_SYS_ERROR) unless unless -f $outputStatsReal; + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $diff_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; } @@ -250,5 +235,5 @@ { my $command = "$difftool -adddiffskyfile -diff_id $diff_id -uri $outputName -path_base $outroot"; - $command .= $stats->cmdflags(); + $command .= " $cmdflags"; $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); $command .= " -hostname $host" if defined $host; Index: /trunk/ippScripts/scripts/register_exp.pl =================================================================== --- /trunk/ippScripts/scripts/register_exp.pl (revision 20100) +++ /trunk/ippScripts/scripts/register_exp.pl (revision 20101) @@ -19,5 +19,4 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Config 1.01 qw( :standard ); @@ -51,61 +50,4 @@ my $DETREND_FLAG = "-end_stage reg"; # Flag to use to mark detrend exposure -# values to extract from output metadata and the stats to calculate -my $STATS = - [ # register imfile - # label STATISTIC CHIPTOOL FLAG - { name => "exp_name", type => "constant", flag => "-exp_name", dtype => "string" }, # File level - { name => "telescope", type => "constant", flag => "-telescope", dtype => "string" }, # File level - { name => "camera", type => "constant", flag => "-inst", dtype => "string" }, # File level - { name => "filelevel", type => "constant", flag => "-filelevel", dtype => "string" }, # File level - { name => "object", type => "constant", flag => "-object", dtype => "string" }, - { name => "exp_type", type => "constant", flag => "-exp_type", dtype => "string" }, # File level - { name => "filter", type => "constant", flag => "-filter", dtype => "string" }, # File level - { name => "comment", type => "constant", flag => "-comment", dtype => "string" }, # ObsComment - { name => "dateobs", type => "constant", flag => "-dateobs", dtype => "string" }, # File level - { name => "ccd_temp", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature - { name => "exp_time", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time - { name => "sat_pixel_frac", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # Fraction of saturated pixels - { name => "airmass", type => "mean", flag => "-airmass", dtype => "float" }, # Airmass - { name => "ra", type => "mean", flag => "-ra", dtype => "float" }, # Right ascension - { name => "decl", type => "mean", flag => "-decl", dtype => "float" }, # Declination - { name => "posang", type => "mean", flag => "-posang", dtype => "float" }, # Position angle - { name => "alt", type => "mean", flag => "-alt", dtype => "float" }, # Altitude - { name => "az", type => "mean", flag => "-az", dtype => "float" }, # Azimuth - { name => "m1_x", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X - { name => "m1_y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y - { name => "m1_z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z - { name => "m1_tip", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP - { name => "m1_tilt", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT - { name => "m2_x", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X - { name => "m2_y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y - { name => "m2_z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z - { name => "m2_tip", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP - { name => "m2_tilt", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT - { name => "env_temperature", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp - { name => "env_humidity", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity - { name => "env_wind_speed", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed - { name => "env_wind_dir", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction - - { name => "teltemp_m1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) - { name => "teltemp_m1cell", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) - { name => "teltemp_m2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C - { name => "teltemp_spider", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) - { name => "teltemp_truss", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C - { name => "teltemp_extra", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) - - { name => "sun_angle", type => "constant", flag => "-sun_angle", dtype => "float" }, # Angle to sun - { name => "sun_alt", type => "constant", flag => "-sun_alt", dtype => "float" }, # Altitude of sun - { name => "moon_angle", type => "constant", flag => "-moon_angle", dtype => "float" }, # Angle to moon - { name => "moon_alt", type => "constant", flag => "-moon_alt", dtype => "float" }, # Altitude of moon - { name => "moon_phase", type => "constant", flag => "-moon_phase", dtype => "float" }, # Phase of moon - - { name => "pon_time", type => "mean", flag => "-pon_time", dtype => "float" }, # time since last power on - { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, # background - { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, # Azimuth - { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, # Azimuth - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for commands we need my $missing_tools; @@ -136,23 +78,35 @@ } - # Parse the output - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - my $metadata = $mdcParser->parse(join "", @$stdout_buf); - unless ($metadata) { - &my_die ("Unable to parse metadata config doc", $exp_id, $PS_EXIT_PROG_ERROR); + # since I can't figure out how to do input and output within PERL, I'm writing to a temp file + my ($statFile, $statName) = tempfile( "/tmp/$exp_id.register.XXXX", UNLINK => !$save_temps ); + print "saving stats to $statName\n"; + foreach my $line (@$stdout_buf) { + print $statFile $line; + } + close $statFile; + + # parse the stats in the metadata file + $command = "$ppStatsFromMetadata $statName - REGISTER_EXP"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform ppStatsFromMetadata: $error_code\n"); + exit($error_code); } - # extract the stats from the metadata - unless ($stats->parse($metadata)) { - &my_die ("Unable to find all values", $exp_id, $PS_EXIT_CONFIG_ERROR); + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; } } -# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type -if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } -if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } -if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } -if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } -if (uc($stats->value_for_flag ("-exp_name")) eq "NULL") { &my_die ("exp_name not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_name, -exp_type +if (uc(&value_for_flag ($cmdflags, "-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc(&value_for_flag ($cmdflags, "-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc(&value_for_flag ($cmdflags, "-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc(&value_for_flag ($cmdflags, "-exp_name")) eq "NULL") { &my_die ("exp_name not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } + +my $exp_type = &value_for_flag ("-exp_type"); +if (uc($exp_type) eq "NULL") { &my_die ("exp_type not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } my $command = "$regtool -addprocessedexp"; @@ -161,7 +115,5 @@ $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; -$command .= $stats->cmdflags(); - -my $exp_type = $stats->value_for_flag ("-exp_type"); +$command .= " $cmdflags"; # Add the detrend flag, if needed @@ -206,4 +158,16 @@ } +sub value_for_flag +{ + my $cmdflags = shift; + my $flag = shift; + + my $value = 0.0; + if ($cmdflags =~ m|$flag|) { + ($value) = $cmdflags =~ m|$flag\s+(\S+)|; + } + $value; +} + sub my_die { Index: /trunk/ippScripts/scripts/register_imfile.pl =================================================================== --- /trunk/ippScripts/scripts/register_imfile.pl (revision 20100) +++ /trunk/ippScripts/scripts/register_imfile.pl (revision 20101) @@ -20,5 +20,4 @@ use PS::IPP::Config 1.01 qw( :standard ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; my $PI = 3.141592653589793238462643383279502; @@ -57,64 +56,9 @@ my $RECIPE = "REGISTER"; # Recipe to use for ppStats -# values to extract from output metadata and the stats to calculate -my $STATS = - [ - # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG - { name => "FILE.LEVEL", type => "constant", flag => "-filelevel", dtype => "string" }, # File level - { name => "CLASS.ID", type => "constant", flag => "-class_id", dtype => "string" }, # Real Class ID - { name => "FPA.OBJECT", type => "constant", flag => "-object", dtype => "string" }, # Object - { name => "FPA.OBSTYPE", type => "constant", flag => "-exp_type", dtype => "string" }, # Exposure type - { name => "FPA.FILTER", type => "constant", flag => "-filter", dtype => "string" }, # Filter used - { name => "FPA.COMMENT", type => "constant", flag => "-comment", dtype => "string" }, # Obs Comment - { name => "FPA.AIRMASS", type => "constant", flag => "-airmass", dtype => "float" }, # Airmass - { name => "FPA.RA", type => "constant", flag => "-ra", dtype => "float" }, # Right ascension - { name => "FPA.DEC", type => "constant", flag => "-decl", dtype => "float" }, # Declination - { name => "FPA.ALT", type => "constant", flag => "-alt", dtype => "float" }, # Altitude - { name => "FPA.AZ", type => "constant", flag => "-az", dtype => "float" }, # Azimuth - { name => "FPA.POSANGLE", type => "constant", flag => "-posang", dtype => "float" }, # Position angle - { name => "FPA.TIME", type => "constant", flag => "-dateobs", dtype => "string" }, # Date of observation (UTC) - { name => "FPA.TELESCOPE", type => "constant", flag => "-telescope", dtype => "string" }, # Telescope - { name => "FPA.CAMERA", type => "constant", flag => "-inst", dtype => "string" }, # Instrument - { name => "FPA.LONGITUDE", type => "constant", flag => "-longitude", dtype => "float" }, # Site longitude - { name => "FPA.LATITUDE", type => "constant", flag => "-latitude", dtype => "float" }, # Site latitude - { name => "FPA.ELEVATION", type => "constant", flag => "-elevation", dtype => "float" }, # Site elevation - { name => "FPA.M1X", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X - { name => "FPA.M1Y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y - { name => "FPA.M1Z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z - { name => "FPA.M1TIP", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP - { name => "FPA.M1TILT", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT - { name => "FPA.M2X", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X - { name => "FPA.M2Y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y - { name => "FPA.M2Z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z - { name => "FPA.M2TIP", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP - { name => "FPA.M2TILT", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT - { name => "FPA.ENV.TEMP", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp - { name => "FPA.ENV.HUMID", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity - { name => "FPA.ENV.WIND", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed - { name => "FPA.ENV.DIR", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction - - { name => "FPA.TELTEMP.M1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) - { name => "FPA.TELTEMP.M1CELL", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) - { name => "FPA.TELTEMP.M2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C - { name => "FPA.TELTEMP.SPIDER", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) - { name => "FPA.TELTEMP.TRUSS", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C - { name => "FPA.TELTEMP.EXTRA", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) - - { name => "FPA.PON.TIME", type => "constant", flag => "-pon_time", dtype => "float" }, # time since last power on - { name => "CHIP.TEMP", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature - { name => "CELL.EXPOSURE", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time - { name => "SAT_PIXEL_FRAC", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # fraction of saturated pixels - { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, - { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, - { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for programs we need my $missing_tools; -my $regtool = can_run('regtool') - or (warn "Can't find regtool" and $missing_tools = 1); -my $ppStats = can_run('ppStats') - or (warn "Can't find ppStats" and $missing_tools = 1); +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $ppStats = can_run('ppStats') or (warn "Can't find ppStats" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { @@ -135,6 +79,5 @@ # Run ppStats on the input file { - # extract the data from the image header; -level is used to get FILE.LEVEL and CLASS.ID - my $command = "$ppStats $uri -recipe PPSTATS $RECIPE -level"; # Command to run ppStats + my $command = "$ppStats $uri -recipe PPSTATS $RECIPE -level | $ppStatsFromMetadata - - REGISTER_IMFILE"; # Command to run ppStats and ppStatsFromMetadata $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -144,16 +87,8 @@ &my_die ("Unable to perform ppStats on exposure id $exp_id: $error_code", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $error_code); } - - # Parse the output - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - my $metadata = $mdcParser->parse(join "", @$stdout_buf); # XXX is this join necessary? - unless ($metadata) { - &my_die ("Unable to parse metadata config doc", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_PROG_ERROR); - } - - # extract the stats from the metadata - unless ($stats->parse($metadata)) { - &my_die ("Unable to find all values", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); - } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; } @@ -162,9 +97,9 @@ # we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type -if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } -if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } -if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } -if (uc($stats->value_for_flag ("-class_id")) eq "NULL") { &my_die ("class_id not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } -if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc(&value_for_flag ($cmdflags, "-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc(&value_for_flag ($cmdflags, "-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc(&value_for_flag ($cmdflags, "-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc(&value_for_flag ($cmdflags, "-class_id")) eq "NULL") { &my_die ("class_id not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc(&value_for_flag ($cmdflags, "-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } my $command = "$regtool -addprocessedimfile"; @@ -175,13 +110,13 @@ $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; -$command .= $stats->cmdflags(); +$command .= " $cmdflags"; # determine solar-system parameters -my $longitude = $stats->value_for_flag ("-longitude"); -my $latitude = $stats->value_for_flag ("-latitude"); -my $elevation = $stats->value_for_flag ("-elevation"); -my $ra = $stats->value_for_flag ("-ra"); -my $dec = $stats->value_for_flag ("-decl"); -my $dateobs = $stats->value_for_flag ("-dateobs"); +my $longitude = &value_for_flag ($cmdflags, "-longitude"); +my $latitude = &value_for_flag ($cmdflags, "-latitude"); +my $elevation = &value_for_flag ($cmdflags, "-elevation"); +my $ra = &value_for_flag ($cmdflags, "-ra"); +my $dec = &value_for_flag ($cmdflags, "-decl"); +my $dateobs = &value_for_flag ($cmdflags, "-dateobs"); # if the needed data is available, pass it to sunmoon: @@ -238,4 +173,16 @@ return @output; } +} + +sub value_for_flag +{ + my $cmdflags = shift; + my $flag = shift; + + my $value = 0.0; + if ($cmdflags =~ m|$flag|) { + ($value) = $cmdflags =~ m|$flag\s+(\S+)|; + } + $value; } Index: /trunk/ippScripts/scripts/stack_skycell.pl =================================================================== --- /trunk/ippScripts/scripts/stack_skycell.pl (revision 20100) +++ /trunk/ippScripts/scripts/stack_skycell.pl (revision 20101) @@ -19,5 +19,4 @@ use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Metadata::List qw( parse_md_list ); use Data::Dumper; @@ -67,35 +66,9 @@ $ipprc->redirect_output($logDest) if $redirect; -my $STATS = - [ - # PPSTATS KEYWORD STATISTIC STACKTOOL FLAG - { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, - { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, - { name => "TIME_STACK", type => "sum", flag => "-dtime_stack", dtype => "float" }, - { name => "TIME_MATCH", type => "mean", flag => "-dtime_match_mean", dtype => "float" }, - { name => "TIME_MATCH", type => "stdev", flag => "-dtime_match_stdev", dtype => "float" }, - { name => "TIME_INITIAL", type => "sum", flag => "-dtime_initial", dtype => "float" }, - { name => "TIME_REJECT", type => "sum", flag => "-dtime_reject", dtype => "float" }, - { name => "TIME_FINAL", type => "sum", flag => "-dtime_final", dtype => "float" }, - { name => "TIME_PHOT", type => "sum", flag => "-dtime_phot", dtype => "float" }, - { name => "STAMP.MEAN", type => "mean", flag => "-match_mean", dtype => "float" }, - { name => "STAMP.MEAN", type => "stdev", flag => "-match_stdev", dtype => "float" }, - { name => "STAMP.RMS", type => "rms", flag => "-match_rms", dtype => "float" }, - { name => "STAMP.NUM", type => "mean", flag => "-stamps_mean", dtype => "float" }, - { name => "STAMP.NUM", type => "stdev", flag => "-stamps_stdev", dtype => "float" }, - { name => "STAMP.NUM", type => "min", flag => "-stamps_min", dtype => "int" }, - { name => "TIME_PHOT", type => "sum", flag => "-dtime_phot", dtype => "float" }, - { name => "REJECT_IMAGES", type => "sum", flag => "-reject_images", dtype => "int" }, - { name => "REJECT_PIXELS", type => "mean", flag => "-reject_pix_mean", dtype => "float" }, - { name => "REJECT_PIXELS", type => "stdev", flag => "-reject_pix_stdev", dtype => "float" }, - { name => "NUM_SOURCES", type => "sum", flag => "-sources", dtype => "int" }, - { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for programs we need my $missing_tools; my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $ppStack = can_run('ppStack') or (warn "Can't find ppStack" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -208,4 +181,6 @@ # doesn't know how to resolve paths. (The information is stored in the configuration file) $configuration = $ipprc->file_resolve($configuration) if ($run_state eq 'update'); + +my $cmdflags; # Perform stacking @@ -244,14 +219,19 @@ if ($do_stats) { - &my_die("Couldn't find expected output file: $outputStats", $stack_id, $PS_EXIT_SYS_ERROR) - unless $ipprc->file_exists($outputStats); - # Get the statistics on the stacked image - my $statsFile; # File handle - open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $stack_id, $PS_EXIT_SYS_ERROR); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - my $metadata = $mdcParser->parse(join "", @contents) or - &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); - $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $stack_id, $PS_EXIT_PROG_ERROR); + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $stack_id, $PS_EXIT_SYS_ERROR) unless unless -f $outputStatsReal; + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - STACK_SKYCELL"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $stack_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; } } @@ -272,5 +252,5 @@ } $command .= " $mode -stack_id $stack_id"; - $command .= $stats->cmdflags() if $do_stats; + $command .= " $cmdflags" if $do_stats; $command .= " -dbname $dbname" if defined $dbname; Index: /trunk/ippScripts/scripts/warp_skycell.pl =================================================================== --- /trunk/ippScripts/scripts/warp_skycell.pl (revision 20100) +++ /trunk/ippScripts/scripts/warp_skycell.pl (revision 20101) @@ -22,5 +22,4 @@ use Pod::Usage qw( pod2usage ); use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config 1.01 qw( :standard ); @@ -63,18 +62,4 @@ $ipprc->redirect_output($logDest) if $redirect; -my $STATS = - [ - # PPSTATS KEYWORD STATISTIC WARPTOOL FLAG - { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, - { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, - { name => "DT_WARP", type => "sum", flag => "-dtime_warp", dtype => "float" }, - { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, - { name => "RANGE.XMIN", type => "mean", flag => "-xmin", dtype => "int" }, - { name => "RANGE.XMAX", type => "mean", flag => "-xmax", dtype => "int" }, - { name => "RANGE.YMIN", type => "mean", flag => "-ymin", dtype => "int" }, - { name => "RANGE.YMAX", type => "mean", flag => "-ymax", dtype => "int" }, - ]; -my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser - # Look for programs we need my $missing_tools; @@ -82,4 +67,5 @@ my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -180,4 +166,5 @@ # Run pswarp +my $cmdflags; my $accept = 1; # Accept the skycell? my $do_stats; @@ -216,18 +203,23 @@ if ($do_stats) { # Check first for the stats file, and if the ACCEPT flag is set. - &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); - # Get the statistics on the warped image - my $statsFile; # File handle - open $statsFile, $ipprc->file_resolve($outputStats) or die "Can't open statistics file $outputStats: $!\n"; - my @contents = <$statsFile>; # Contents of file - close $statsFile; - my $contents = join "", @contents; - - my $metadata = $mdcParser->parse($contents) - or &my_die("Unable to parse metadata config", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); - $accept = metadataLookupBool($metadata, "ACCEPT"); - - # $accept is set above based on the fraction of lit pixels - # XXX for some files, there may not be enough stars to find a good psf. these should be dropped as well + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal; + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYFILE"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + + # the stats includes ACCEPT as a boolean: convert the T/F value to 0/1 + my $acceptFlag = &value_for_flag ($cmdflags, "-accept"); + $accept = ($acceptFlag = "T") ? 1 : 0; if ($accept && !$ipprc->file_exists($outputPSF)) { $accept = 0; @@ -235,13 +227,8 @@ if ($accept) { - $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); - &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - # &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); - # &my_die("Couldn't find expected output file: $outputBin1", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); - # &my_die("Couldn't find expected output file: $outputBin2", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); } @@ -256,5 +243,5 @@ $command .= " -uri $outputImage" if $accept; $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $command .= $stats->cmdflags() if $accept; + $command .= " $cmdflags" if $accept; $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; @@ -285,4 +272,16 @@ } } +} + +sub value_for_flag +{ + my $cmdflags = shift; + my $flag = shift; + + my $value = 0.0; + if ($cmdflags =~ m|$flag|) { + ($value) = $cmdflags =~ m|$flag\s+(\S+)|; + } + $value; }