Changeset 32556
- Timestamp:
- Oct 14, 2011, 4:41:16 PM (15 years ago)
- Location:
- trunk
- Files:
-
- 1 added
- 7 edited
-
ippScripts/scripts/videophot_process.pl (modified) (9 diffs)
-
ippTasks/vp.pro (modified) (2 diffs)
-
ippTools/share/Makefile.am (modified) (1 diff)
-
ippTools/share/vptool_processedcell.sql (added)
-
psvideophot/src/dumpvideo.c (modified) (4 diffs)
-
psvideophot/src/psvideoLoop.c (modified) (6 diffs)
-
psvideophot/src/psvideophotArguments.c (modified) (1 diff)
-
psvideophot/src/psvideophotParseCamera.c (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/videophot_process.pl
r32546 r32556 22 22 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); 23 23 use Pod::Usage qw( pod2usage ); 24 use File::Temp qw( tempfile ); 25 use File::Basename qw( basename ); 24 26 25 27 # Look for programs we need … … 27 29 my $vptool = can_run('vptool') or (warn "Can't find vptool" and $missing_tools = 1); 28 30 my $psvideophot = can_run('psvideophot') or (warn "Can't find psvideophot" and $missing_tools = 1); 31 my $dumpvideo = can_run('dumpvideo') or (warn "Can't find dumpvideo" and $missing_tools = 1); 29 32 my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1); 33 my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); 30 34 if ($missing_tools) { 31 35 warn("Can't find required tools."); 32 36 exit($PS_EXIT_CONFIG_ERROR); 33 37 } 34 my ($vp_id, $camera, $outroot, $dest_id, $dbname, $verbose, $no_update, $no_op, $redirect); 38 my ($vp_id, $camera, $outroot, $dest_id, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps); 39 my ($product, $ds_dbname, $ds_dbhost); 35 40 36 41 GetOptions( … … 39 44 'outroot=s' => \$outroot, 40 45 'dest_id=s' => \$dest_id, 46 'product=s' => \$product, 47 'ds_dbname=s' => \$ds_dbname, 48 'ds_dbhost=s' => \$ds_dbhost, 41 49 'dbname|d=s' => \$dbname, 42 50 'verbose' => \$verbose, 51 'save-temps' => \$save_temps, 43 52 'no-update' => \$no_update, # Don't update the database? 44 53 'no-op' => \$no_op, # Don't do any operations? … … 54 63 and defined $outroot; 55 64 65 pod2usage( 66 -msg => " --product -ds_dbname and --dsdbhost are required if --dest_id", 67 -exitval => 3, 68 ) if ($dest_id and !( defined $product and defined $ds_dbname and defined $ds_dbhost)); 69 56 70 57 71 $no_update = 1 if $no_op; 72 $vptool .= " -dbname $dbname" if $dbname; 58 73 59 74 my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration … … 66 81 print "Starting script $0 on $host at $date\n\n"; 67 82 } 68 69 $vptool .= " -dbname $dbname" if $dbname;70 83 71 84 # Get list of chips with pending video cells … … 90 103 } 91 104 105 92 106 foreach my $file (@$files) { 93 107 my $class_id = $file->{class_id}; … … 110 124 foreach my $cell_id (@video_cells) { 111 125 my $path_base = "$outroot.$class_id.$cell_id"; 112 my $command = "$psvideophot $path_base"; 126 my $output = $ipprc->filename("PSVIDEOPHOT.OUTPUT", $path_base); 127 my $command = "$psvideophot $output"; 113 128 $command .= " -file $uri"; 129 $command .= " -class_id $class_id"; 114 130 $command .= " -cell_id $cell_id"; 115 131 my $vpstart = DateTime->now->mjd; … … 119 135 unless ($success) { 120 136 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 121 &my_die("Unable to perform p pSub: $error_code", $vp_id, $error_code);137 &my_die("Unable to perform psvideophot: $error_code", $vp_id, $error_code); 122 138 } 123 124 # check_output($outputStats, 1);125 139 } else { 126 140 print "Not executing: $command\n"; 127 141 } 142 # dump the video cell and the video table to a file 143 unless ($no_op) { 144 my $output = $path_base . ".fits"; 145 my $extname = "$cell_id"; 146 my $command = "$dumpvideo $uri $extname $output -includetable"; 147 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 148 run(command => $command, verbose => $verbose); 149 unless ($success) { 150 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 151 &my_die("Unable to perform dumpvideo: $error_code", $vp_id, $error_code); 152 } 153 } else { 154 print "Not executing: $command\n"; 155 } 156 128 157 unless ($no_update) { 129 158 my $command = "$vptool -addprocessedcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id"; … … 142 171 } 143 172 } 144 # TODO: if dest_id build results fileset and post it in the given destination on the data store 173 if ($dest_id) { 174 my $command = "$vptool -processedcell -vp_id $vp_id"; 175 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 176 run(command => $command, verbose => $verbose); 177 unless ($success) { 178 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 179 &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code); 180 } 181 182 my $output = join "", @$stdout_buf; 183 my $cells; 184 if ($output) { 185 my $metadata = $mdcParser->parse($output) or 186 &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR); 187 $cells = parse_md_list($metadata) or 188 &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR); 189 } 190 my ($reglist, $reglistName) = tempfile("/tmp/filelist.$vp_id.XXXX", UNLINK => !$save_temps); 191 192 if ($redirect) { 193 print $reglist "$logDest|||text|\n"; 194 } 195 my @exts = qw( vpt fits ); 196 my @types = qw( table fits ); 197 foreach my $cell (@$cells) { 198 my $path_base = $cell->{path_base}; 199 for (my $i = 0; $i < scalar @exts; $i++) { 200 my $file = "$path_base.$exts[$i]"; 201 my $type = $types[$i]; 202 203 print $reglist "$file|||$type|\n"; 204 } 205 206 } 207 close $reglist or &my_die("failed to close $reglistName", $vp_id, $PS_EXIT_UNKNOWN_ERROR); 208 unless ($no_update or $no_op) { 209 my $fileset = basename($outroot); 210 my $command = "$dsreg --add $fileset --product $product --type dump --list $reglistName"; 211 $command .= " --abspath --link"; 212 $command .= " --dbname $ds_dbname --dbhost $ds_dbhost"; 213 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 214 run(command => $command, verbose => $verbose); 215 unless ($success) { 216 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 217 &my_die("Unable to perform $command error_code: $error_code", $vp_id, $error_code); 218 } 219 } 220 } 145 221 146 222 unless ($no_update) { -
trunk/ippTasks/vp.pro
r32546 r32556 146 146 book getword vpPendingRun $pageName workdir -var WORKDIR 147 147 book getword vpPendingRun $pageName dest_id -var DEST_ID 148 book getword vpPendingRun $pageName dest_name -var DEST_NAME 149 book getword vpPendingRun $pageName ds_dbname -var DS_DBNAME 150 book getword vpPendingRun $pageName ds_dbhost -var DS_DBHOST 148 151 book getword vpPendingRun $pageName dbname -var DBNAME 149 152 … … 152 155 # set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR 153 156 154 sprintf outroot "%s/%s.vp " $WORKDIR $EXP_TAG157 sprintf outroot "%s/%s.vp.%d" $WORKDIR $EXP_TAG $VP_ID 155 158 156 159 stdout $LOGSUBDIR/vp.log 157 160 stderr $LOGSUBDIR/vp.log 158 161 159 $run = videophot_process.pl --vp_id $VP_ID --outroot $outroot --redirect-output --camera $CAMERA --dest_id $DEST_ID 162 $run = videophot_process.pl --vp_id $VP_ID --outroot $outroot --redirect-output --camera $CAMERA 163 if ($DEST_ID) 164 $run = $run --dest_id $DEST_ID --product $DEST_NAME --ds_dbname $DS_DBNAME --ds_dbhost $DS_DBHOST 165 end 160 166 add_standard_args run 161 167 -
trunk/ippTools/share/Makefile.am
r32510 r32556 430 430 vptool_pendingimfile.sql \ 431 431 vptool_pendingrun.sql \ 432 vptool_processedcell.sql \ 432 433 vptool_revertrun.sql 433 434 -
trunk/psvideophot/src/dumpvideo.c
r32553 r32556 2 2 #include <assert.h> 3 3 #include <pslib.h> 4 #include <psmodules.h> 4 5 5 6 int main (int argc, char **argv) { … … 9 10 exit (2); 10 11 } 12 13 pmConfig *config = pmConfigRead(&argc, argv, NULL); 14 if (config == NULL) { 15 psErrorStackPrint(stderr, "Can't find site configuration!\n"); 16 exit(EXIT_FAILURE); 17 } 18 19 11 20 12 21 char *input = argv[1]; // Input file name … … 19 28 } 20 29 21 psFits *infile = psFitsOpen (input, "r"); // Input file 30 psString inputResolved = pmConfigConvertFilename(input, config, false, false); 31 psFits *infile = psFitsOpen (inputResolved, "r"); // Input file 22 32 if (!infile) { 23 33 fprintf (stderr, "Unable to open input file %s", input); … … 25 35 } 26 36 27 psFits *outfile = psFitsOpen (output, "w"); // Output file 37 psString outputResolved = pmConfigConvertFilename(output, config, true, true); 38 psFits *outfile = psFitsOpen (outputResolved, "w"); // Output file 28 39 if (!outfile) { 29 40 fprintf (stderr, "Unable to open output file %s", output); -
trunk/psvideophot/src/psvideoLoop.c
r32553 r32556 58 58 ESCAPE("load failure for FPA"); 59 59 } 60 61 psArray *output = NULL; 60 62 61 63 pmChip *chip; // Chip from FPA … … 136 138 } 137 139 psFree(video_extension_name); 138 psMetadataAddArray(cell->analysis, PS_LIST_TAIL, "VIDEO_DATA", 0, "", video_table); 140 // psMetadataAddArray(cell->analysis, PS_LIST_TAIL, "VIDEO_DATA", 0, "", video_table); 141 142 output = (psArray *) psMemIncrRefCounter(video_table); 139 143 140 144 if (cell->readouts->n != video_table->n) { … … 195 199 double cnpix2 = psMetadataLookupS32(&status, row, "cnpix2"); 196 200 201 // avoid errors from unused variables 197 202 (void) frame_start; (void) frame_complete; 198 203 (void) rowpre; (void) cnpix1; (void) cnpix2; (void) fwhm_x; (void) fwhm_y; (void) fwhm; … … 206 211 double star_readout_y = (centroid_y - cnpix2) / yBin; 207 212 213 #ifdef notdef 208 214 printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.6f %6.3f %6.3f %6.3f\n", 209 215 frame, flux, snr, sky, star_readout_x, star_readout_y, frame_complete - frame_start, 210 216 fwhm, fwhm_x, fwhm_y); 211 217 #endif 212 218 psStatsInit(stats); 213 219 … … 221 227 double stddev = stats->sampleStdev; 222 228 double median = stats->sampleMedian; 229 double vpFlux = 0; 230 double vpSum = 0; 231 double xy[2] = {0, 0}; 223 232 if (stddev > threshold) { 224 233 225 double sum = 0; 226 // XXX: use a point data type 227 double xy[2]; 228 Centering(readout->image, EDGE, &sum, xy, median); 229 double ourFlux; 230 ourFlux = SubSum(readout->image, xy, median); 231 232 printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.2f\n", 233 frame, ourFlux, ourFlux/stddev, median, xy[0], xy[1], stddev); 234 Centering(readout->image, EDGE, &vpSum, xy, median); 235 236 vpFlux = SubSum(readout->image, xy, median); 237 238 // printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.2f\n", 239 // frame, vpFlux, vpFlux/median, median, xy[0], xy[1], stddev); 234 240 } else { 235 printf("%4d %9.2f %9.2f %8.2f\n", 236 frame, 0.0, stddev, median); 237 } 241 // printf("%4d %9.2f %9.2f %8.2f\n", 242 // frame, 0.0, stddev, median); 243 } 244 if (frame % 60 == 0) { 245 printf("frame flux sky snr X Y dt fwhm fwhm_x fwhm_y vp_flux vp_median vp_sum vp_stddev vp_x vp_y\n"); 246 } 247 printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.6f %6.3f %6.3f %6.3f %9.2f %9.2f %9.2f %9.2f %.1f %.1f\n", 248 frame, flux, sky, snr, star_readout_x, star_readout_y, frame_complete - frame_start, 249 fwhm, fwhm_x, fwhm_y, 250 vpFlux, median, vpSum, stddev, xy[0], xy[1]); 251 psMetadataAddF32(row, PS_LIST_TAIL, "vp_flux", 0, "vp flux", vpFlux); 252 psMetadataAddF32(row, PS_LIST_TAIL, "vp_median", 0, "vp meidan", median); 253 psMetadataAddF32(row, PS_LIST_TAIL, "vp_stddev", 0, "vp meidan", stddev); 254 psMetadataAddF32(row, PS_LIST_TAIL, "vp_sum", 0, "vp sum of source", vpSum); 255 psMetadataAddF32(row, PS_LIST_TAIL, "vp_x", 0, "centroid x from vp", xy[0]); 256 psMetadataAddF32(row, PS_LIST_TAIL, "vp_y", 0, "centroid y from vp", xy[1]); 238 257 } 239 258 psLogMsg ("psvideophotLoop", 5, "Done processing video cell"); … … 262 281 psFree(view); 263 282 264 // Dump configuration 265 psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG"); 266 if (dump_file) { 267 if (!pmConfigDump(config, dump_file)) { 268 ESCAPE("Unable to dump configuration."); 269 } 270 } 283 psString outputName = psMetadataLookupStr(&status, config->arguments, "OUTPUT"); 284 psString realName = pmConfigConvertFilename(outputName, config, true, true); 285 if (!realName) { 286 psError(PS_ERR_UNKNOWN, false, "failed to convert filename for %s", outputName); 287 ESCAPE("cannot save output"); 288 } 289 psFits *outFits = psFitsOpen(realName, "w"); 290 if (!outFits) { 291 psError(PS_ERR_UNKNOWN, false, "failed to open output filename for %s", outputName); 292 ESCAPE(" "); 293 } 294 295 psMetadata *outputHeader = lastHDU ? lastHDU->header : NULL; 296 if (!psFitsWriteTable(outFits, outputHeader, output, "videophotemetry")) { 297 psError(PS_ERR_UNKNOWN, false, "failed to write videophotometry table"); 298 ESCAPE("cannot write output"); 299 } 300 psFitsClose(outFits); 271 301 272 302 return true; -
trunk/psvideophot/src/psvideophotArguments.c
r32553 r32556 180 180 181 181 // Add the input and output images (which remain on the command-line) to the arguments list 182 psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]);182 psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output table", argv[1]); 183 183 184 184 return config; -
trunk/psvideophot/src/psvideophotParseCamera.c
r31290 r32556 7 7 psvideophotOptions *psvideophotParseCamera(pmConfig *config) 8 8 { 9 bool status = false;10 11 9 if (!psvideophotDefineFile(config, NULL, "PSVIDEOPHOT.INPUT", "INPUT", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_NONE)) { 12 10 psError(PS_ERR_IO, false, "Can't find an input image source"); … … 16 14 psAssert(input, "We just put it there!"); 17 15 16 #ifdef notyet 18 17 // add recipe options supplied on command line 18 bool status = false; 19 19 psMetadata *recipe = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME); 20 #endif 20 21 21 22 // parse the options from the metadata format to the psvideophotOptions structure 22 23 psvideophotOptions *options = psvideophotOptionsParse(config); 23 24 24 // the following are defined from the argument list, if given,25 // otherwise they revert to the config information26 // not all input or output images are used in a given recipe27 28 int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS");29 if (nThreads > 0) {30 int nScanRows = psMetadataLookupS32(&status, recipe, "SCAN.ROWS");31 pmDetrendSetThreadTasks(nScanRows);32 }33 34 // the following files are output targets35 pmFPAfile *outImage = pmFPAfileDefineOutput(config, input->fpa, "PSVIDEOPHOT.OUTPUT");36 if (!outImage) {37 psError(PS_ERR_IO, false, _("Unable to generate output file from PSVIDEOPHOT.OUTPUT"));38 psFree(options);39 return NULL;40 }41 if (outImage->type != PM_FPA_FILE_OBJ) {42 psError(PS_ERR_IO, true, "PSVIDEOPHOT.OUTPUT is not of type OBJ");43 psFree(options);44 return NULL;45 }46 47 // XXX should these be bound explicitly to the outImage->fpa rather than the input->fpa?48 pmFPAfile *chipImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PSVIDEOPHOT.CHIP");49 if (!chipImage) {50 psError(PS_ERR_IO, false, _("Unable to generate new file from PSVIDEOPHOT.CHIP"));51 psFree(options);52 return NULL;53 }54 if (chipImage->type != PM_FPA_FILE_OBJ) {55 psError(PS_ERR_IO, true, "PSVIDEOPHOT.CHIP is not of type OBJ");56 psFree(options);57 return NULL;58 }59 60 61 // chipImage -> psphotInput (pmFPAfileDefineFromFile) : fpa is constructed62 // psphotInput -> psphotOutput (pmFPAfileDefineOutputFromFile) : fpa is equated63 // psphotOutput -> psastroInput (pmFPAfileDefineInput) : fpa is ref-copied64 // psastroInput -> psastroModel (pmFPAfileDefineFromArgs : fpa is ref-copied65 // psastroInput -> psastroModel (pmFPAfileDefineFromConf : fpa is constructed66 // psastroInput -> psastroModel (pmFPAfileDefineFromDetDB : fpa is constructed (pmDetrendSelect uses input concepts )67 68 // For photometry, we operate on the chip-mosaicked image69 // we create a copy of the mosaicked image for psphot so we can write out a clean image70 71 72 // this file is just used as a carrier; output files (eg, PSPHOT.RESID) are defined by73 // psphotDefineFiles74 pmFPAfile *psphotInput = pmFPAfileDefineFromFile (config, chipImage, 1, 1, "PSPHOT.INPUT");75 PS_ASSERT (psphotInput, false);76 77 // specify the number of psphot input images78 psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "number of inputs", 1);79 80 // define associated psphot input/output files81 if (!psphotDefineFiles (config, psphotInput)) {82 psError(PSPHOT_ERR_CONFIG, false,83 "Trouble defining the additional input/output files for psphot");84 return false;85 }86 87 88 // For photometry, we operate on the chip-mosaicked image89 90 91 // save any of these files?92 outImage->save = options->BaseFITS;93 94 95 chipImage->save = options->ChipFITS;96 97 98 99 // outImage is used as a carrier: input to chipImage -> require the data to remain at the CHIP level100 outImage->freeLevel = PS_MIN(outImage->freeLevel, PM_FPA_LEVEL_CHIP);101 outImage->dataLevel = outImage->freeLevel;102 outImage->fileLevel = PS_MIN(outImage->fileLevel, outImage->dataLevel);103 104 // outMask and outVariance must be freed at the same level as outImage (all freed by pmFPAFreeData)105 106 107 // the input data is the same as the outImage data : force the free levels to match108 input->freeLevel = PS_MIN(outImage->freeLevel, input->freeLevel);109 110 // define the binned target files (which may just be carriers for some camera configurations)111 112 // Chip selection: turn on only the chips specified (pass status to suppress missing-key log msg)113 char *chipLine = psMetadataLookupStr(&status, config->arguments, "CHIP_SELECTIONS");114 psArray *chips = psStringSplitArray(chipLine, ",", false);115 if (chips->n > 0) {116 pmFPASelectChip (input->fpa, -1, true); // deselect all chips117 for (int i = 0; i < chips->n; i++) {118 int chipNum = atoi(chips->data[i]);119 if (! pmFPASelectChip(input->fpa, chipNum, false)) {120 psError(PS_ERR_IO, false, "Chip number %d doesn't exist in camera.\n", chipNum);121 psFree(options);122 return false;123 }124 }125 }126 psFree (chips);127 128 if (psMetadataLookupBool(NULL, config->arguments, "INPUT_IS_FRINGE")) {129 // It's a fringe file, so change the file type130 input->type = PM_FPA_FILE_FRINGE;131 outImage->type = PM_FPA_FILE_FRINGE;132 }133 if (psMetadataLookupBool(NULL, config->arguments, "INPUT_IS_DARK")) {134 // It's a dark file, so change the file type135 input->type = PM_FPA_FILE_DARK;136 outImage->type = PM_FPA_FILE_DARK;137 // Turn off compression --- there are just too many nasties that can happen138 psFree(outImage->compression);139 outImage->compression = NULL;140 psFree(outImage->options);141 outImage->options = NULL;142 }143 144 // Turn off mask and variance output if we're not doing anything interesting145 146 if (psTraceGetLevel("psvideophot.config") > 0) {147 // Get a look inside all the files.148 psMetadataIterator *filesIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, NULL);149 psMetadataItem *item; // Item from iteration150 fprintf(stderr, "Files:\n");151 while ((item = psMetadataGetAndIncrement(filesIter))) {152 pmFPAfile *file = item->data.V; // File of interest153 fprintf(stderr, "%s: %p %p %p (%p) %p\n", file->name,154 file->src, file->fpa,155 file->camera, file->fpa->camera, file->format);156 }157 psFree(filesIter);158 }159 160 // Change the input dark type between the old (IMAGE) and new (multi-DARK).161 // Hopefully this is a temporary change until we all move over to using the new dark types162 bool mdok; // Status of MD lookup163 if (options->doDark && psMetadataLookupBool(&mdok, recipe, "OLDDARK")) {164 pmFPAfile *dark = psMetadataLookupPtr(NULL, config->files, "PSVIDEOPHOT.DARK");165 assert(dark);166 dark->type = PM_FPA_FILE_IMAGE;167 }168 169 25 return (options); 170 26 }
Note:
See TracChangeset
for help on using the changeset viewer.
