Index: /trunk/ippScripts/scripts/videophot_process.pl =================================================================== --- /trunk/ippScripts/scripts/videophot_process.pl (revision 32555) +++ /trunk/ippScripts/scripts/videophot_process.pl (revision 32556) @@ -22,4 +22,6 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename ); # Look for programs we need @@ -27,10 +29,13 @@ my $vptool = can_run('vptool') or (warn "Can't find vptool" and $missing_tools = 1); my $psvideophot = can_run('psvideophot') or (warn "Can't find psvideophot" and $missing_tools = 1); +my $dumpvideo = can_run('dumpvideo') or (warn "Can't find dumpvideo" and $missing_tools = 1); my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1); +my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); exit($PS_EXIT_CONFIG_ERROR); } -my ($vp_id, $camera, $outroot, $dest_id, $dbname, $verbose, $no_update, $no_op, $redirect); +my ($vp_id, $camera, $outroot, $dest_id, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($product, $ds_dbname, $ds_dbhost); GetOptions( @@ -39,6 +44,10 @@ 'outroot=s' => \$outroot, 'dest_id=s' => \$dest_id, + 'product=s' => \$product, + 'ds_dbname=s' => \$ds_dbname, + 'ds_dbhost=s' => \$ds_dbhost, 'dbname|d=s' => \$dbname, 'verbose' => \$verbose, + 'save-temps' => \$save_temps, 'no-update' => \$no_update, # Don't update the database? 'no-op' => \$no_op, # Don't do any operations? @@ -54,6 +63,12 @@ and defined $outroot; +pod2usage( + -msg => " --product -ds_dbname and --dsdbhost are required if --dest_id", + -exitval => 3, + ) if ($dest_id and !( defined $product and defined $ds_dbname and defined $ds_dbhost)); + $no_update = 1 if $no_op; +$vptool .= " -dbname $dbname" if $dbname; my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -66,6 +81,4 @@ print "Starting script $0 on $host at $date\n\n"; } - -$vptool .= " -dbname $dbname" if $dbname; # Get list of chips with pending video cells @@ -90,4 +103,5 @@ } + foreach my $file (@$files) { my $class_id = $file->{class_id}; @@ -110,6 +124,8 @@ foreach my $cell_id (@video_cells) { my $path_base = "$outroot.$class_id.$cell_id"; - my $command = "$psvideophot $path_base"; + my $output = $ipprc->filename("PSVIDEOPHOT.OUTPUT", $path_base); + my $command = "$psvideophot $output"; $command .= " -file $uri"; + $command .= " -class_id $class_id"; $command .= " -cell_id $cell_id"; my $vpstart = DateTime->now->mjd; @@ -119,11 +135,24 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppSub: $error_code", $vp_id, $error_code); + &my_die("Unable to perform psvideophot: $error_code", $vp_id, $error_code); } - - # check_output($outputStats, 1); } else { print "Not executing: $command\n"; } + # dump the video cell and the video table to a file + unless ($no_op) { + my $output = $path_base . ".fits"; + my $extname = "$cell_id"; + my $command = "$dumpvideo $uri $extname $output -includetable"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dumpvideo: $error_code", $vp_id, $error_code); + } + } else { + print "Not executing: $command\n"; + } + unless ($no_update) { my $command = "$vptool -addprocessedcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id"; @@ -142,5 +171,52 @@ } } -# TODO: if dest_id build results fileset and post it in the given destination on the data store +if ($dest_id) { + my $command = "$vptool -processedcell -vp_id $vp_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code); + } + + my $output = join "", @$stdout_buf; + my $cells; + if ($output) { + my $metadata = $mdcParser->parse($output) or + &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR); + $cells = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR); + } + my ($reglist, $reglistName) = tempfile("/tmp/filelist.$vp_id.XXXX", UNLINK => !$save_temps); + + if ($redirect) { + print $reglist "$logDest|||text|\n"; + } + my @exts = qw( vpt fits ); + my @types = qw( table fits ); + foreach my $cell (@$cells) { + my $path_base = $cell->{path_base}; + for (my $i = 0; $i < scalar @exts; $i++) { + my $file = "$path_base.$exts[$i]"; + my $type = $types[$i]; + + print $reglist "$file|||$type|\n"; + } + + } + close $reglist or &my_die("failed to close $reglistName", $vp_id, $PS_EXIT_UNKNOWN_ERROR); + unless ($no_update or $no_op) { + my $fileset = basename($outroot); + my $command = "$dsreg --add $fileset --product $product --type dump --list $reglistName"; + $command .= " --abspath --link"; + $command .= " --dbname $ds_dbname --dbhost $ds_dbhost"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command error_code: $error_code", $vp_id, $error_code); + } + } +} unless ($no_update) { Index: /trunk/ippTasks/vp.pro =================================================================== --- /trunk/ippTasks/vp.pro (revision 32555) +++ /trunk/ippTasks/vp.pro (revision 32556) @@ -146,4 +146,7 @@ book getword vpPendingRun $pageName workdir -var WORKDIR book getword vpPendingRun $pageName dest_id -var DEST_ID + book getword vpPendingRun $pageName dest_name -var DEST_NAME + book getword vpPendingRun $pageName ds_dbname -var DS_DBNAME + book getword vpPendingRun $pageName ds_dbhost -var DS_DBHOST book getword vpPendingRun $pageName dbname -var DBNAME @@ -152,10 +155,13 @@ # set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR - sprintf outroot "%s/%s.vp" $WORKDIR $EXP_TAG + sprintf outroot "%s/%s.vp.%d" $WORKDIR $EXP_TAG $VP_ID stdout $LOGSUBDIR/vp.log stderr $LOGSUBDIR/vp.log - $run = videophot_process.pl --vp_id $VP_ID --outroot $outroot --redirect-output --camera $CAMERA --dest_id $DEST_ID + $run = videophot_process.pl --vp_id $VP_ID --outroot $outroot --redirect-output --camera $CAMERA + if ($DEST_ID) + $run = $run --dest_id $DEST_ID --product $DEST_NAME --ds_dbname $DS_DBNAME --ds_dbhost $DS_DBHOST + end add_standard_args run Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 32555) +++ /trunk/ippTools/share/Makefile.am (revision 32556) @@ -430,4 +430,5 @@ vptool_pendingimfile.sql \ vptool_pendingrun.sql \ + vptool_processedcell.sql \ vptool_revertrun.sql Index: /trunk/ippTools/share/vptool_processedcell.sql =================================================================== --- /trunk/ippTools/share/vptool_processedcell.sql (revision 32556) +++ /trunk/ippTools/share/vptool_processedcell.sql (revision 32556) @@ -0,0 +1,22 @@ +SELECT + vpProcessedCell.*, + vpRun.state, + vpRun.label, + vpRun.data_group, + vpRun.outroot, + rawExp.exp_id, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.filter, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + rawExp.dateobs, + rawExp.obs_mode, + rawExp.comment, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM vpRun + JOIN vpProcessedCell USING(vp_id) + JOIN rawExp USING(exp_id) Index: /trunk/psvideophot/src/dumpvideo.c =================================================================== --- /trunk/psvideophot/src/dumpvideo.c (revision 32555) +++ /trunk/psvideophot/src/dumpvideo.c (revision 32556) @@ -2,4 +2,5 @@ #include #include +#include int main (int argc, char **argv) { @@ -9,4 +10,12 @@ exit (2); } + + pmConfig *config = pmConfigRead(&argc, argv, NULL); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + char *input = argv[1]; // Input file name @@ -19,5 +28,6 @@ } - psFits *infile = psFitsOpen (input, "r"); // Input file + psString inputResolved = pmConfigConvertFilename(input, config, false, false); + psFits *infile = psFitsOpen (inputResolved, "r"); // Input file if (!infile) { fprintf (stderr, "Unable to open input file %s", input); @@ -25,5 +35,6 @@ } - psFits *outfile = psFitsOpen (output, "w"); // Output file + psString outputResolved = pmConfigConvertFilename(output, config, true, true); + psFits *outfile = psFitsOpen (outputResolved, "w"); // Output file if (!outfile) { fprintf (stderr, "Unable to open output file %s", output); Index: /trunk/psvideophot/src/psvideoLoop.c =================================================================== --- /trunk/psvideophot/src/psvideoLoop.c (revision 32555) +++ /trunk/psvideophot/src/psvideoLoop.c (revision 32556) @@ -58,4 +58,6 @@ ESCAPE("load failure for FPA"); } + + psArray *output = NULL; pmChip *chip; // Chip from FPA @@ -136,5 +138,7 @@ } psFree(video_extension_name); - psMetadataAddArray(cell->analysis, PS_LIST_TAIL, "VIDEO_DATA", 0, "", video_table); + // psMetadataAddArray(cell->analysis, PS_LIST_TAIL, "VIDEO_DATA", 0, "", video_table); + + output = (psArray *) psMemIncrRefCounter(video_table); if (cell->readouts->n != video_table->n) { @@ -195,4 +199,5 @@ double cnpix2 = psMetadataLookupS32(&status, row, "cnpix2"); + // avoid errors from unused variables (void) frame_start; (void) frame_complete; (void) rowpre; (void) cnpix1; (void) cnpix2; (void) fwhm_x; (void) fwhm_y; (void) fwhm; @@ -206,8 +211,9 @@ double star_readout_y = (centroid_y - cnpix2) / yBin; +#ifdef notdef printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.6f %6.3f %6.3f %6.3f\n", frame, flux, snr, sky, star_readout_x, star_readout_y, frame_complete - frame_start, fwhm, fwhm_x, fwhm_y); - +#endif psStatsInit(stats); @@ -221,19 +227,32 @@ double stddev = stats->sampleStdev; double median = stats->sampleMedian; + double vpFlux = 0; + double vpSum = 0; + double xy[2] = {0, 0}; if (stddev > threshold) { - double sum = 0; - // XXX: use a point data type - double xy[2]; - Centering(readout->image, EDGE, &sum, xy, median); - double ourFlux; - ourFlux = SubSum(readout->image, xy, median); - - printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.2f\n", - frame, ourFlux, ourFlux/stddev, median, xy[0], xy[1], stddev); + Centering(readout->image, EDGE, &vpSum, xy, median); + + vpFlux = SubSum(readout->image, xy, median); + +// printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.2f\n", +// frame, vpFlux, vpFlux/median, median, xy[0], xy[1], stddev); } else { - printf("%4d %9.2f %9.2f %8.2f\n", - frame, 0.0, stddev, median); - } +// printf("%4d %9.2f %9.2f %8.2f\n", +// frame, 0.0, stddev, median); + } + if (frame % 60 == 0) { + printf("frame flux sky snr X Y dt fwhm fwhm_x fwhm_y vp_flux vp_median vp_sum vp_stddev vp_x vp_y\n"); + } + printf("%4d %9.2f %9.2f %9.2f %.1f %.1f %9.6f %6.3f %6.3f %6.3f %9.2f %9.2f %9.2f %9.2f %.1f %.1f\n", + frame, flux, sky, snr, star_readout_x, star_readout_y, frame_complete - frame_start, + fwhm, fwhm_x, fwhm_y, + vpFlux, median, vpSum, stddev, xy[0], xy[1]); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_flux", 0, "vp flux", vpFlux); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_median", 0, "vp meidan", median); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_stddev", 0, "vp meidan", stddev); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_sum", 0, "vp sum of source", vpSum); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_x", 0, "centroid x from vp", xy[0]); + psMetadataAddF32(row, PS_LIST_TAIL, "vp_y", 0, "centroid y from vp", xy[1]); } psLogMsg ("psvideophotLoop", 5, "Done processing video cell"); @@ -262,11 +281,22 @@ psFree(view); - // Dump configuration - psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG"); - if (dump_file) { - if (!pmConfigDump(config, dump_file)) { - ESCAPE("Unable to dump configuration."); - } - } + psString outputName = psMetadataLookupStr(&status, config->arguments, "OUTPUT"); + psString realName = pmConfigConvertFilename(outputName, config, true, true); + if (!realName) { + psError(PS_ERR_UNKNOWN, false, "failed to convert filename for %s", outputName); + ESCAPE("cannot save output"); + } + psFits *outFits = psFitsOpen(realName, "w"); + if (!outFits) { + psError(PS_ERR_UNKNOWN, false, "failed to open output filename for %s", outputName); + ESCAPE(" "); + } + + psMetadata *outputHeader = lastHDU ? lastHDU->header : NULL; + if (!psFitsWriteTable(outFits, outputHeader, output, "videophotemetry")) { + psError(PS_ERR_UNKNOWN, false, "failed to write videophotometry table"); + ESCAPE("cannot write output"); + } + psFitsClose(outFits); return true; Index: /trunk/psvideophot/src/psvideophotArguments.c =================================================================== --- /trunk/psvideophot/src/psvideophotArguments.c (revision 32555) +++ /trunk/psvideophot/src/psvideophotArguments.c (revision 32556) @@ -180,5 +180,5 @@ // Add the input and output images (which remain on the command-line) to the arguments list - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output table", argv[1]); return config; Index: /trunk/psvideophot/src/psvideophotParseCamera.c =================================================================== --- /trunk/psvideophot/src/psvideophotParseCamera.c (revision 32555) +++ /trunk/psvideophot/src/psvideophotParseCamera.c (revision 32556) @@ -7,6 +7,4 @@ psvideophotOptions *psvideophotParseCamera(pmConfig *config) { - bool status = false; - if (!psvideophotDefineFile(config, NULL, "PSVIDEOPHOT.INPUT", "INPUT", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_NONE)) { psError(PS_ERR_IO, false, "Can't find an input image source"); @@ -16,155 +14,13 @@ psAssert(input, "We just put it there!"); +#ifdef notyet // add recipe options supplied on command line + bool status = false; psMetadata *recipe = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME); +#endif // parse the options from the metadata format to the psvideophotOptions structure psvideophotOptions *options = psvideophotOptionsParse(config); - // the following are defined from the argument list, if given, - // otherwise they revert to the config information - // not all input or output images are used in a given recipe - - int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); - if (nThreads > 0) { - int nScanRows = psMetadataLookupS32(&status, recipe, "SCAN.ROWS"); - pmDetrendSetThreadTasks(nScanRows); - } - - // the following files are output targets - pmFPAfile *outImage = pmFPAfileDefineOutput(config, input->fpa, "PSVIDEOPHOT.OUTPUT"); - if (!outImage) { - psError(PS_ERR_IO, false, _("Unable to generate output file from PSVIDEOPHOT.OUTPUT")); - psFree(options); - return NULL; - } - if (outImage->type != PM_FPA_FILE_OBJ) { - psError(PS_ERR_IO, true, "PSVIDEOPHOT.OUTPUT is not of type OBJ"); - psFree(options); - return NULL; - } - - // XXX should these be bound explicitly to the outImage->fpa rather than the input->fpa? - pmFPAfile *chipImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PSVIDEOPHOT.CHIP"); - if (!chipImage) { - psError(PS_ERR_IO, false, _("Unable to generate new file from PSVIDEOPHOT.CHIP")); - psFree(options); - return NULL; - } - if (chipImage->type != PM_FPA_FILE_OBJ) { - psError(PS_ERR_IO, true, "PSVIDEOPHOT.CHIP is not of type OBJ"); - psFree(options); - return NULL; - } - - - // chipImage -> psphotInput (pmFPAfileDefineFromFile) : fpa is constructed - // psphotInput -> psphotOutput (pmFPAfileDefineOutputFromFile) : fpa is equated - // psphotOutput -> psastroInput (pmFPAfileDefineInput) : fpa is ref-copied - // psastroInput -> psastroModel (pmFPAfileDefineFromArgs : fpa is ref-copied - // psastroInput -> psastroModel (pmFPAfileDefineFromConf : fpa is constructed - // psastroInput -> psastroModel (pmFPAfileDefineFromDetDB : fpa is constructed (pmDetrendSelect uses input concepts ) - - // For photometry, we operate on the chip-mosaicked image - // we create a copy of the mosaicked image for psphot so we can write out a clean image - - - // this file is just used as a carrier; output files (eg, PSPHOT.RESID) are defined by - // psphotDefineFiles - pmFPAfile *psphotInput = pmFPAfileDefineFromFile (config, chipImage, 1, 1, "PSPHOT.INPUT"); - PS_ASSERT (psphotInput, false); - - // specify the number of psphot input images - psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "number of inputs", 1); - - // define associated psphot input/output files - if (!psphotDefineFiles (config, psphotInput)) { - psError(PSPHOT_ERR_CONFIG, false, - "Trouble defining the additional input/output files for psphot"); - return false; - } - - - // For photometry, we operate on the chip-mosaicked image - - - // save any of these files? - outImage->save = options->BaseFITS; - - - chipImage->save = options->ChipFITS; - - - - // outImage is used as a carrier: input to chipImage -> require the data to remain at the CHIP level - outImage->freeLevel = PS_MIN(outImage->freeLevel, PM_FPA_LEVEL_CHIP); - outImage->dataLevel = outImage->freeLevel; - outImage->fileLevel = PS_MIN(outImage->fileLevel, outImage->dataLevel); - - // outMask and outVariance must be freed at the same level as outImage (all freed by pmFPAFreeData) - - - // the input data is the same as the outImage data : force the free levels to match - input->freeLevel = PS_MIN(outImage->freeLevel, input->freeLevel); - - // define the binned target files (which may just be carriers for some camera configurations) - - // Chip selection: turn on only the chips specified (pass status to suppress missing-key log msg) - char *chipLine = psMetadataLookupStr(&status, config->arguments, "CHIP_SELECTIONS"); - psArray *chips = psStringSplitArray(chipLine, ",", false); - if (chips->n > 0) { - pmFPASelectChip (input->fpa, -1, true); // deselect all chips - for (int i = 0; i < chips->n; i++) { - int chipNum = atoi(chips->data[i]); - if (! pmFPASelectChip(input->fpa, chipNum, false)) { - psError(PS_ERR_IO, false, "Chip number %d doesn't exist in camera.\n", chipNum); - psFree(options); - return false; - } - } - } - psFree (chips); - - if (psMetadataLookupBool(NULL, config->arguments, "INPUT_IS_FRINGE")) { - // It's a fringe file, so change the file type - input->type = PM_FPA_FILE_FRINGE; - outImage->type = PM_FPA_FILE_FRINGE; - } - if (psMetadataLookupBool(NULL, config->arguments, "INPUT_IS_DARK")) { - // It's a dark file, so change the file type - input->type = PM_FPA_FILE_DARK; - outImage->type = PM_FPA_FILE_DARK; - // Turn off compression --- there are just too many nasties that can happen - psFree(outImage->compression); - outImage->compression = NULL; - psFree(outImage->options); - outImage->options = NULL; - } - - // Turn off mask and variance output if we're not doing anything interesting - - if (psTraceGetLevel("psvideophot.config") > 0) { - // Get a look inside all the files. - psMetadataIterator *filesIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, NULL); - psMetadataItem *item; // Item from iteration - fprintf(stderr, "Files:\n"); - while ((item = psMetadataGetAndIncrement(filesIter))) { - pmFPAfile *file = item->data.V; // File of interest - fprintf(stderr, "%s: %p %p %p (%p) %p\n", file->name, - file->src, file->fpa, - file->camera, file->fpa->camera, file->format); - } - psFree(filesIter); - } - - // Change the input dark type between the old (IMAGE) and new (multi-DARK). - // Hopefully this is a temporary change until we all move over to using the new dark types - bool mdok; // Status of MD lookup - if (options->doDark && psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { - pmFPAfile *dark = psMetadataLookupPtr(NULL, config->files, "PSVIDEOPHOT.DARK"); - assert(dark); - dark->type = PM_FPA_FILE_IMAGE; - } - return (options); }