Index: /branches/czw_branch/20100427/ippTools/share/magictool_definebyquery_select.sql =================================================================== --- /branches/czw_branch/20100427/ippTools/share/magictool_definebyquery_select.sql (revision 28041) +++ /branches/czw_branch/20100427/ippTools/share/magictool_definebyquery_select.sql (revision 28042) @@ -35,7 +35,8 @@ JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) LEFT JOIN (SELECT magic_id, exp_id, label - FROM magicRun + FROM magicRun -- rerun hook %s ) AS oldMagicRun Index: /branches/czw_branch/20100427/ippTools/src/magictool.c =================================================================== --- /branches/czw_branch/20100427/ippTools/src/magictool.c (revision 28041) +++ /branches/czw_branch/20100427/ippTools/src/magictool.c (revision 28042) @@ -132,4 +132,5 @@ psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, queryWhere, "-select_filter", "rawExp.filter", "=="); // Get list of exposures ready to magic Index: /branches/czw_branch/20100427/ippTools/src/magictoolConfig.c =================================================================== --- /branches/czw_branch/20100427/ippTools/src/magictoolConfig.c (revision 28041) +++ /branches/czw_branch/20100427/ippTools/src/magictoolConfig.c (revision 28042) @@ -58,4 +58,5 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", 0, "select filter", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false); Index: /branches/czw_branch/20100427/ippconfig/gpc1/camera.config =================================================================== --- /branches/czw_branch/20100427/ippconfig/gpc1/camera.config (revision 28041) +++ /branches/czw_branch/20100427/ippconfig/gpc1/camera.config (revision 28042) @@ -153,4 +153,4 @@ BURNTOOL.STATE.GOOD.UPDATE S16 13 # for upddate processing accept earlier version -FOV F32 15000 +FOV F32 15000 # Field of view of unvignetted region in FPA pixels. NPIX_INTERCHIP S32 0 Index: /branches/czw_branch/20100427/ippconfig/simtest/camera.config =================================================================== --- /branches/czw_branch/20100427/ippconfig/simtest/camera.config (revision 28041) +++ /branches/czw_branch/20100427/ippconfig/simtest/camera.config (revision 28042) @@ -78,4 +78,2 @@ PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.NAME} # Rule for generating photcode -FOV F32 15000 -NPIX_INTERCHIP S32 0 Index: /branches/czw_branch/20100427/ppImage/src/ppImageOptions.c =================================================================== --- /branches/czw_branch/20100427/ppImage/src/ppImageOptions.c (revision 28041) +++ /branches/czw_branch/20100427/ppImage/src/ppImageOptions.c (revision 28042) @@ -110,10 +110,4 @@ options->normClass = NULL; // per-class normalizations refer to this class - // Mask stats options - options->maskstat_static = 0; // static masks - options->maskstat_dynamic = 0; // dynamic masks - options->maskstat_magic = 0; // magic masks - options->maskstat_advisory = 0; // advisory masks - return options; } Index: /branches/czw_branch/20100427/ppSub/src/ppSubLoop.c =================================================================== --- /branches/czw_branch/20100427/ppSub/src/ppSubLoop.c (revision 28041) +++ /branches/czw_branch/20100427/ppSub/src/ppSubLoop.c (revision 28042) @@ -146,5 +146,4 @@ // Do Mask Stats { - printf("HEREHEREHERE\n"); pmFPAview *view = ppSubViewReadout(); // View to readout if (!ppSubMaskStats(config, view,data->stats)) { Index: /branches/czw_branch/20100427/psLib/src/imageops/psImageStats.c =================================================================== --- /branches/czw_branch/20100427/psLib/src/imageops/psImageStats.c (revision 28041) +++ /branches/czw_branch/20100427/psLib/src/imageops/psImageStats.c (revision 28042) @@ -97,14 +97,7 @@ psVectorMaskType *data = junkMask->data.PS_TYPE_VECTOR_MASK_DATA; - psU8* imdata = junkData->data.U8; for (int row = 0, nVect = 0; row < numRows; row++) { for (int col = 0; col < numCols; col++, nVect++) { data[nVect] = (mask->data.PS_TYPE_IMAGE_MASK_DATA[row][col] & maskVal); - - if ((imdata[nVect] > 100000)|| - (imdata[nVect] != imdata[nVect])|| - (imdata[nVect] < -100000)) { - data[nVect] &= maskVal; - } } } Index: /branches/czw_branch/20100427/psModules/src/detrend/pmMaskStats.c =================================================================== --- /branches/czw_branch/20100427/psModules/src/detrend/pmMaskStats.c (revision 28042) +++ /branches/czw_branch/20100427/psModules/src/detrend/pmMaskStats.c (revision 28042) @@ -0,0 +1,121 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include +#include +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmFPAview.h" +#include "pmFPAfile.h" +#include "pmFPAfileIO.h" +#include "pmFPAAstrometry.h" + + +# define ESCAPE { \ + psError(PS_ERR_UNKNOWN, false, "I/O failure in pmMaskStats"); \ + psFree (view); \ + return false; \ + } + + + + +bool pmFPAMaskStats(pmFPA *fpa, pmConfig *config) { + PS_ASSERT_PTR_NON_NULL(fpa, false); + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status; + + psU16 staticMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.STATIC"); + psU16 magicMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.MAGIC"); + psU16 dynamicMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.DYNAMIC"); + psU16 advisoryMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.ADVISORY"); + + psS32 Npix_valid = 0; + psS32 Npix_static = 0; + psS32 Npix_magic = 0; + psS32 Npix_dynamic = 0; + psS32 Npix_advisory = 0; + + pmChip *chip; + pmCell *cell; + pmReadout *readout; + pmFPAview *view = pmFPAviewAlloc(0); + while ((chip = pmFPAviewNextChip(view, fpa, 1)) != NULL) { + if (!chip->process || !chip->file_exists) { continue; } + if (!chip->fromFPA) { continue; } + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE; + + while ((cell = pmFPAviewNextCell(view, fpa, 1)) != NULL) { + if (!cell->process || !cell->file_exists) {continue; } + while ((readout = pmFPAviewNextReadout(view, fpa, 1)) != NULL) { + if (!readout->data_exists) {continue; } + + psImage *mask = readout->mask; + if (!pmSingleImageMaskStats(mask,&Npix_valid,&Npix_static,&Npix_magic, + &Npix_dynamic,&Npix_advisory, + staticMaskVal,magicMaskVal, + dynamicMaskVal,advisoryMaskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate masks for readout."); + return(false); + } + psMetadataAddS32(readout->analysis, PS_LIST_TAIL,"MASKFRAC_NPIX", 0, + "Number of valid pixels", Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + } + } + } + return(true); +} + + + +bool pmSingleImageMaskStats(psImage *mask, + psS32 *Npix_valid, psS32 *Npix_static, psS32 *Npix_magic, + psS32 *Npix_dynamic, psS32 *Npix_advisory, + psU16 staticMaskVal, psU16 magicMaskVal, + psU16 dynamicMaskVal, psU16 advisoryMaskVal) { + PS_ASSERT_IMAGE_NON_NULL(mask, false); + *Npix_valid = 0; + *Npix_static = 0; + *Npix_magic = 0; + *Npix_dynamic = 0; + *Npix_advisory = 0; + + psImageMaskType **maskData = mask->data.PS_TYPE_IMAGE_MASK_DATA; + psWarning("HERE: %ld %d %d\n",(long) maskData, mask->numRows,mask->numCols); + for (int i = 0; i < mask->numRows - 1; i++) { + for (int j = 0; j < mask->numCols - 1; j++) { + *Npix_valid += 1; + if (maskData[i][j] & staticMaskVal) { + *Npix_static += 1; + continue; + } + if (maskData[i][j] & dynamicMaskVal) { + *Npix_dynamic += 1; + continue; + } + if (maskData[i][j] & magicMaskVal) { + *Npix_magic += 1; + continue; + } + if (maskData[i][j] & advisoryMaskVal) { + *Npix_advisory += 1; + continue; + } + } + } + return(true); +} + Index: /branches/czw_branch/20100427/psModules/src/detrend/pmMaskStats.h =================================================================== --- /branches/czw_branch/20100427/psModules/src/detrend/pmMaskStats.h (revision 28042) +++ /branches/czw_branch/20100427/psModules/src/detrend/pmMaskStats.h (revision 28042) @@ -0,0 +1,14 @@ +#ifndef PM_MASK_STATS_H +#define PM_MASK_STATS_H + +bool pmFPAMaskStats(pmFPA *fpa, pmConfig *config); +bool pmSingleImageMaskStats(psImage *mask, + psS32 *Npix_valid, psS32 *Npix_static, psS32 *Npix_magic, + psS32 *Npix_dynamic, psS32 *Npix_advisory, + psU16 staticMaskVal, psU16 magicMaskVal, + psU16 dynamicMaskVal, psU16 advisoryMaskVal); + + +//bool pmMaskStats(pmFPA *fpa, pmConfig *config, psMetadata *results); + +#endif Index: /branches/czw_branch/20100427/pstamp/scripts/Makefile.am =================================================================== --- /branches/czw_branch/20100427/pstamp/scripts/Makefile.am (revision 28041) +++ /branches/czw_branch/20100427/pstamp/scripts/Makefile.am (revision 28042) @@ -23,4 +23,5 @@ detectability_respond.pl \ detect_query_read \ + detect_query_create \ detect_response_create \ dquery_finish.pl \ Index: /branches/czw_branch/20100427/pstamp/scripts/detect_query_create =================================================================== --- /branches/czw_branch/20100427/pstamp/scripts/detect_query_create (revision 28042) +++ /branches/czw_branch/20100427/pstamp/scripts/detect_query_create (revision 28042) @@ -0,0 +1,287 @@ +#!/usr/bin/env perl + +# create a MOPS_DETECTABILITY_QUERY fits table from a text description file + +use warnings; +use strict; + +use Astro::FITS::CFITSIO qw( :constants ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for output table + +my ( $input, # Name of input text file + $output, # Name of output table + $query_id, + $nostage, + ); + +GetOptions( + 'input|i=s' => \$input, + 'output|o=s' => \$output, + 'query_id|q=s' => \$query_id, + 'nostage' => \$nostage, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --input --output", + -exitval => 3) unless defined $input and defined $output; + +# The header kewords +my $header = [ + { name => 'QUERY_ID', + writetype => TSTRING, + comment => 'MOPS Query ID for this batch query', + value => undef + }, + { name => 'EXTVER', + writetype => TSTRING, + comment => 'Extension version', + value => undef + }, + { name => 'FPA_ID', + writetype => TSTRING, + comment => 'orginal FPA_ID used at ingest', + value => undef + }, + { name => 'MJD-OBS', + writetype => TDOUBLE, + comment => 'starting time of the exposure, MJD', + value => undef + }, + { name => 'FILTER', + writetype => TSTRING, + comment => 'effective filter use for the exposure', + value => undef + }, + { name => 'OBSCODE', + writetype => TSTRING, + comment => 'site identifier (MPC observatory code)', + value => undef + }, + { name => 'STAGE', + writetype => TSTRING, + comment => 'processing stage to examine', + value => undef + } +]; +unless(defined($nostage)) { + push @{ $header }, { name => 'STAGE', + writetype => TSTRING, + comment => 'processing stage to examine', + value => undef + }; +} + +# Specification of columns to write +my $columns = [ + # matching rownum from detectability original request + { name => 'ROWNUM', type => '20A', writetype => TSTRING }, + # coordinate at start of exposure, in degrees + { name => 'RA1_DEG', type => 'D', writetype => TDOUBLE }, + # coordinate at start of exposure, in degrees + { name => 'DEC1_DEG', type => 'D', writetype => TDOUBLE }, + # coordinate at end of exposure, in degrees + { name => 'RA2_DEG', type => 'D', writetype => TDOUBLE }, + # coordinate at end of exposure, in degrees + { name => 'DEC2_DEG', type => 'D', writetype => TDOUBLE }, + # apparent magnitude + { name => 'MAG', type => 'D', writetype => TDOUBLE }, +]; + +my $in; +if ($input eq '-') { + $in = \*STDIN; +} else { + open $in, "<$input" or die "cannot open $input for reading"; +} + +my @colData; +foreach (@$columns) { + push @colData, []; +} + + +my $numRows = read_data_for_table($in,'\s+', \@colData, $header); +if (!$numRows) { + print STDERR "no data in $input\n"; + exit 1; +} + +# overwrite the QUERY_ID value from the input file with the command +# line argument + +if ($query_id) { + $header->[0]->{value} = $query_id; +} + +my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header); + +exit $status; + +# XXXXX: This should be in a module +# two utility functions that may be used to create a FITS binary +# table from hashes describing the header keywords and columns + +# read_table_description reads the data for a table from a simple text file +# make_fits_table writes out the table to a named file + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +# A function to build a fits binary table from supplied data +# +sub make_fits_table { + my $output = shift; # name of output file + my $extname = shift; # extension name + my $numRows = shift; # number of rows in the table + my $colData = shift; # ref to array of arrays containing the data for each column + my $columns = shift; # ref to array of column descriptions (each a hash) + # with keys: name, type, and writetype + my $header = shift; # ref to array of header keyword descriptions - each a hash + # with keys: name, name, writetype, comment, and value + my $status = 0; + + die "incorrect arguments" if !defined($columns); + # note $header can be nil + + # build arrays for cfitsio + my @colNames; # Names of columns + my @colTypes; # Types of columns + my @colWriteType; # type to use to write + + foreach my $colSpec ( @$columns) { + push @colNames, $colSpec->{name}; + push @colTypes, $colSpec->{type}; + push @colWriteType, $colSpec->{writetype}; + } + + if (-e $output) { + unlink "$output" or die "failed to remove existing $output"; + } + + my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); # Output file handle + check_fitsio( $status ); + + $outFits->create_img( 16, 0, undef, $status ); + check_fitsio( $status ); + + # Create the table + + $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, + \@colNames, \@colTypes, undef, $extname, $status ); + check_fitsio( $status ); + + # if header keyword descriptions were provided add them + if ($header) { + foreach my $headerword ( @$header ) { + my $value = $headerword->{value}; + unless (defined $value) { + print "Can't find header keyword $headerword\n"; + next; + } + # zap quotation marks + $value =~ s/\'//g; + my $name = $headerword->{name}; + my $type = $headerword->{writetype}; + my $comment = $headerword->{comment}; + $outFits->write_key( $type, $name, $value, $comment, $status ); + check_fitsio( $status ); + } + } + + + for (my $i = 0; $i < scalar @colNames; $i++) { + my $writeType = $colWriteType[$i]; + $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData->[$i], $status ); + check_fitsio( $status ); + } + + $outFits->close_file( $status ); + + return 0; + +} # end of sub make_fits_table + + + +# read the table contents from a file +# +# input text file format: +# lines that begin with '#' are comment lines and are skipped. +# other lines are data. Each data line is split into fields with the +# provided separator +# +# if $header is not null header the first non-commented line is read to +# fill the value for each header keyword. The number of fields must match +# the number of keywords. +# +# Following the optional header data, each data line contains data for each +# row in the table. The number of fields must match the number of column +# arrays provided. + +sub read_data_for_table { + my $in = shift; # input file handle + my $sep = shift; # string containing field separator + my $colData = shift; # reference to an array of arrays for the data + my $header = shift; # rerence to array of header keyword descriptions + + my $line_num = 0; + + # read data for header if any data is expected + if ($header) { + my $nhead = @$header; + while (my $line = <$in>) { + $line_num++; + chomp $line; + next if !$line; # skip blank lines + next if ($line =~ /^#/); # skip comment lines + my @vals = split /$sep/, $line; + my $nvals = @vals; + die "number of header columns in input $nvals does not equal expected number of header words $nhead" + if (@vals != @$header); + + for (my $i=0; $i < @$header; $i++) { + $header->[$i]->{value} = $vals[$i]; + } + + last; # only one header line + } + } + + my $row_num = 0; + my $ncols = @$colData; + while (my $line = <$in>) { + chomp $line; + $line_num++; + next if !$line; # skip blank lines + next if ($line =~ /^#/); # skip comment lines + + my @vals = split /$sep/, $line; + my $nvals = @vals; + die "number of columns $nvals in input does not equal expected number of header " + . " words $ncols on line $line_num" if ($nvals != $ncols); + + for (my $col = 0; $col < @$colData; $col++) { + $colData->[$col]->[$row_num] = $vals[$col]; + } + $row_num++; + } + + # we return the number of rows read + return $row_num; +} + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + die "CFITSIO error: $msg\n"; + } +}