Index: trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 28041) +++ trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 28043) @@ -742,4 +742,5 @@ # print "Time to run dvoImagesAtCoords: $dtime_dvo\n"; unless ($success) { + my $result_code = $error_code >> 8; if ($result_code == $PSTAMP_NO_OVERLAP) { Index: trunk/dbconfig/cam.md =================================================================== --- trunk/dbconfig/cam.md (revision 28041) +++ trunk/dbconfig/cam.md (revision 28043) @@ -14,4 +14,10 @@ end_stage STR 64 # Key magicked S64 0 + software_ver STR 16 + maskfrac_npix S32 0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 note STR 255 END @@ -91,4 +97,10 @@ path_base STR 255 fault S16 0 # Key NOT NULL + software_ver STR 16 + maskfrac_npix S32 0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 quality S16 0 END Index: trunk/dbconfig/changes.txt =================================================================== --- trunk/dbconfig/changes.txt (revision 28041) +++ trunk/dbconfig/changes.txt (revision 28043) @@ -1605,4 +1605,6 @@ ALTER TABLE distTarget ADD KEY(stage), ADD KEY(dist_group), ADD KEY(filter), ADD KEY(state); +-- Version 1.1.67 + CREATE TABLE Label ( label VARCHAR(64), @@ -1616,2 +1618,64 @@ ALTER TABLE pstampRequest ADD COLUMN outdir VARCHAR(64) after uri; + +-- Version 1.1.68 +ALTER TABLE chipRun ADD COLUMN software_ver VARCHAR(16) after magicked; +ALTER TABLE chipRun ADD COLUMN maskfrac_npix INT after software_ver; +ALTER TABLE chipRun ADD COLUMN maskfrac_static FLOAT after maskfrac_npix; +ALTER TABLE chipRun ADD COLUMN maskfrac_dynamic FLOAT after maskfrac_static; +ALTER TABLE chipRun ADD COLUMN maskfrac_magic FLOAT after maskfrac_dynamic; +ALTER TABLE chipRun ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + +ALTER TABLE chipProcessedImfile ADD COLUMN software_ver VARCHAR(16) after magicked; +ALTER TABLE chipProcessedImfile ADD COLUMN maskfrac_npix INT after software_ver; +ALTER TABLE chipProcessedImfile ADD COLUMN maskfrac_static FLOAT after maskfrac_npix; +ALTER TABLE chipProcessedImfile ADD COLUMN maskfrac_dynamic FLOAT after maskfrac_static; +ALTER TABLE chipProcessedImfile ADD COLUMN maskfrac_magic FLOAT after maskfrac_dynamic; +ALTER TABLE chipProcessedImfile ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + +ALTER TABLE camRun ADD COLUMN software_ver VARCHAR(16) after magicked; +ALTER TABLE camRun ADD COLUMN maskfrac_npix INT after software_ver; +ALTER TABLE camRun ADD COLUMN maskfrac_static FLOAT after maskfrac_npix; +ALTER TABLE camRun ADD COLUMN maskfrac_dynamic FLOAT after maskfrac_static; +ALTER TABLE camRun ADD COLUMN maskfrac_magic FLOAT after maskfrac_dynamic; +ALTER TABLE camRun ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + +ALTER TABLE camProcessedExp ADD COLUMN software_ver VARCHAR(16) after fault; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_npix INT after software_ver; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_static FLOAT after maskfrac_npix; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_dynamic FLOAT after maskfrac_static; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_magic FLOAT after maskfrac_dynamic; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + +ALTER TABLE warpRun ADD COLUMN software_ver VARCHAR(16) after magicked; +ALTER TABLE warpRun ADD COLUMN maskfrac_npix INT after software_ver; +ALTER TABLE warpRun ADD COLUMN maskfrac_static FLOAT after maskfrac_npix; +ALTER TABLE warpRun ADD COLUMN maskfrac_dynamic FLOAT after maskfrac_static; +ALTER TABLE warpRun ADD COLUMN maskfrac_magic FLOAT after maskfrac_dynamic; +ALTER TABLE warpRun ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + +ALTER TABLE warpSkyfile ADD COLUMN software_ver VARCHAR(16) after magicked; +ALTER TABLE warpSkyfile ADD COLUMN maskfrac_npix INT after software_ver; +ALTER TABLE warpSkyfile ADD COLUMN maskfrac_static FLOAT after maskfrac_npix; +ALTER TABLE warpSkyfile ADD COLUMN maskfrac_dynamic FLOAT after maskfrac_static; +ALTER TABLE warpSkyfile ADD COLUMN maskfrac_magic FLOAT after maskfrac_dynamic; +ALTER TABLE warpSkyfile ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + +ALTER TABLE stackRun ADD COLUMN software_ver VARCHAR(16) after filter; + +ALTER TABLE stackSumSkyfile ADD COLUMN software_ver VARCHAR(16) after fault; + +ALTER TABLE diffRun ADD COLUMN software_ver VARCHAR(16) after magicked; +ALTER TABLE diffRun ADD COLUMN maskfrac_npix INT after software_ver; +ALTER TABLE diffRun ADD COLUMN maskfrac_static FLOAT after maskfrac_npix; +ALTER TABLE diffRun ADD COLUMN maskfrac_dynamic FLOAT after maskfrac_static; +ALTER TABLE diffRun ADD COLUMN maskfrac_magic FLOAT after maskfrac_dynamic; +ALTER TABLE diffRun ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + +ALTER TABLE diffSkyfile ADD COLUMN software_ver VARCHAR(16) after magicked; +ALTER TABLE diffSkyfile ADD COLUMN maskfrac_npix INT after software_ver; +ALTER TABLE diffSkyfile ADD COLUMN maskfrac_static FLOAT after maskfrac_npix; +ALTER TABLE diffSkyfile ADD COLUMN maskfrac_dynamic FLOAT after maskfrac_static; +ALTER TABLE diffSkyfile ADD COLUMN maskfrac_magic FLOAT after maskfrac_dynamic; +ALTER TABLE diffSkyfile ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + Index: trunk/dbconfig/chip.md =================================================================== --- trunk/dbconfig/chip.md (revision 28041) +++ trunk/dbconfig/chip.md (revision 28043) @@ -14,4 +14,10 @@ end_stage STR 64 # Key magicked S64 0 + software_ver STR 16 + maskfrac_npix S32 0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 note STR 255 END @@ -93,4 +99,10 @@ quality S16 0 magicked S64 0 + software_ver STR 16 + maskfrac_npix S32 0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 END Index: trunk/dbconfig/diff.md =================================================================== --- trunk/dbconfig/diff.md (revision 28041) +++ trunk/dbconfig/diff.md (revision 28043) @@ -15,4 +15,10 @@ exposure BOOL f magicked S64 0 + software_ver STR 16 + maskfrac_npix S32 0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 diff_mode S16 0 note STR 255 @@ -62,3 +68,9 @@ quality S16 0 magicked S64 0 + software_ver STR 16 + maskfrac_npix S32 0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 END Index: trunk/dbconfig/stack.md =================================================================== --- trunk/dbconfig/stack.md (revision 28041) +++ trunk/dbconfig/stack.md (revision 28043) @@ -14,4 +14,5 @@ tess_id STR 64 # Key filter STR 64 + software_ver STR 16 note STR 255 END @@ -49,4 +50,5 @@ good_frac F32 0.0 # Key fault S16 0 # Key + software_ver STR 16 quality S16 0 END Index: trunk/dbconfig/warp.md =================================================================== --- trunk/dbconfig/warp.md (revision 28041) +++ trunk/dbconfig/warp.md (revision 28043) @@ -25,4 +25,10 @@ registered TAI NULL magicked S64 0 + software_ver STR 16 + maskfrac_npix S32 0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 note STR 255 END @@ -62,4 +68,10 @@ quality S16 0 magicked S64 0 + software_ver STR 16 + maskfrac_npix S32 0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 END Index: trunk/ippMonitor/def/detProcessedExp_failure.d =================================================================== --- trunk/ippMonitor/def/detProcessedExp_failure.d (revision 28041) +++ trunk/ippMonitor/def/detProcessedExp_failure.d (revision 28043) @@ -13,5 +13,5 @@ ARGS ARG1 basename=$detProcessedExp.path_base -OP OP1 $detProcessedExp.bg / $detProcessedExp.bg_stdev +OP OP1 $detProcessedExp.bg_stdev > 0.0 ? $detProcessedExp.bg / $detProcessedExp.bg_stdev : NAN # field size format name show link to extras Index: trunk/ippMonitor/def/rawImfile_failed.d =================================================================== --- trunk/ippMonitor/def/rawImfile_failed.d (revision 28041) +++ trunk/ippMonitor/def/rawImfile_failed.d (revision 28043) @@ -1,5 +1,5 @@ TABLE rawImfile TITLE Failed Raw Image Files -FILE rawImfile.php +FILE rawImfile_failed.php MENU ipp.load.dat Index: trunk/ippMonitor/raw/getimage.php =================================================================== --- trunk/ippMonitor/raw/getimage.php (revision 28041) +++ trunk/ippMonitor/raw/getimage.php (revision 28043) @@ -8,4 +8,6 @@ // global $BINDIR; // global $SITE; +// global $PSCONFDIR +// global $PSCONFIG ### these need to be set to the correct locations!! @@ -18,4 +20,7 @@ } + +putenv("PSCONFDIR=$PSCONFDIR"); +putenv("PSCONFIG=$PSCONFIG"); putenv("PERL5LIB=$PERLLIB:"); $PATH = getenv("PATH"); Index: trunk/ippMonitor/raw/ipp.php =================================================================== --- trunk/ippMonitor/raw/ipp.php (revision 28041) +++ trunk/ippMonitor/raw/ipp.php (revision 28043) @@ -722,5 +722,9 @@ global $BINDIR; global $SITE; - + global $PSCONFDIR; + global $PSCONFIG; + + putenv("PSCONFDIR=$PSCONFDIR"); + putenv("PSCONFIG=$PSCONFIG"); putenv("PERL5LIB=$PERLLIB:"); $PATH = getenv("PATH"); Index: trunk/ippMonitor/raw/site.php.in =================================================================== --- trunk/ippMonitor/raw/site.php.in (revision 28041) +++ trunk/ippMonitor/raw/site.php.in (revision 28043) @@ -11,4 +11,7 @@ $SITE = "@SITE@"; +$PSCONFDIR = "@PSCONFDIR@"; +$PSCONFIG = "@PSCONFIG@"; + $DBI = "@DBI@"; ?> Index: trunk/ippScripts/scripts/camera_exp.pl =================================================================== --- trunk/ippScripts/scripts/camera_exp.pl (revision 28041) +++ trunk/ippScripts/scripts/camera_exp.pl (revision 28043) @@ -280,4 +280,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + print STDERR (join "\n", @$stderr_buf); &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code); } Index: trunk/ippScripts/scripts/diff_skycell.pl =================================================================== --- trunk/ippScripts/scripts/diff_skycell.pl (revision 28041) +++ trunk/ippScripts/scripts/diff_skycell.pl (revision 28043) @@ -292,5 +292,5 @@ $command .= " -recipe PPSUB $recipe_ppSub"; $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -recipe PPSTATS DIFFSTATS"; $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; Index: trunk/ippScripts/scripts/stack_skycell.pl =================================================================== --- trunk/ippScripts/scripts/stack_skycell.pl (revision 28041) +++ trunk/ippScripts/scripts/stack_skycell.pl (revision 28043) @@ -232,5 +232,5 @@ $command .= " -recipe PPSUB $recipe_ppSub"; $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -recipe PPSTATS WARPSTATS" if $do_stats;; + $command .= " -recipe PPSTATS STACKSTATS" if $do_stats;; $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; Index: trunk/ippToPsps/src/ippToPspsVersion.c =================================================================== --- trunk/ippToPsps/src/ippToPspsVersion.c (revision 28041) +++ trunk/ippToPsps/src/ippToPspsVersion.c (revision 28043) @@ -58,4 +58,6 @@ psString source = ippToPsps_Source(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "IPP2PS_V", PS_META_REPLACE, NULL, IPPTOPSPS_VERSION); + psStringPrepend(&version, "ippToPsps version: "); psStringPrepend(&source, "ippToPsps source: "); Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 28041) +++ trunk/ippTools/share/Makefile.am (revision 28043) @@ -28,4 +28,5 @@ chiptool_change_imfile_data_state.sql \ chiptool_completely_processed_exp.sql \ + chiptool_coalesce_run.sql \ chiptool_definecopy.sql \ chiptool_donecleanup.sql \ @@ -97,4 +98,5 @@ difftool_change_run_state.sql \ difftool_completed_runs.sql \ + difftool_coalesce_run.sql \ difftool_definewarpstack_part1.sql \ difftool_definewarpstack_part2.sql \ @@ -298,4 +300,5 @@ warptool_finished_run_select.sql \ warptool_finish_run.sql \ + warptool_coalesce_run.sql \ warptool_imfile.sql \ warptool_listrun.sql \ Index: trunk/ippTools/share/camtool_queue_chip_id.sql =================================================================== --- trunk/ippTools/share/camtool_queue_chip_id.sql (revision 28041) +++ trunk/ippTools/share/camtool_queue_chip_id.sql (revision 28043) @@ -17,4 +17,10 @@ '%s', -- end_stage %lld, -- magicked + '%s', -- software ver + '%s', -- maskfrac_npix + '%s', -- _static + '%s', -- _dynamic + '%s', -- _magic + '%s', -- _advisory '%s' -- note FROM chipRun Index: trunk/ippTools/share/chiptool_coalesce_run.sql =================================================================== --- trunk/ippTools/share/chiptool_coalesce_run.sql (revision 28043) +++ trunk/ippTools/share/chiptool_coalesce_run.sql (revision 28043) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from chipProcessedImfile where data_state = 'full' AND fault = 0 AND chip_id = %lld Index: trunk/ippTools/share/difftool_coalesce_run.sql =================================================================== --- trunk/ippTools/share/difftool_coalesce_run.sql (revision 28043) +++ trunk/ippTools/share/difftool_coalesce_run.sql (revision 28043) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from diffSkyfile where diff_id = %lld Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 28041) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 28043) @@ -276,4 +276,10 @@ end_stage VARCHAR(64), magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, note VARCHAR(255), PRIMARY KEY(chip_id), @@ -359,4 +365,10 @@ fault SMALLINT NOT NULL, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, PRIMARY KEY(chip_id, exp_id, class_id), KEY(data_state), @@ -387,4 +399,10 @@ end_stage VARCHAR(64), magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, note VARCHAR(255), PRIMARY KEY(cam_id), @@ -464,4 +482,10 @@ quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT NOT NULL, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, PRIMARY KEY(cam_id), KEY(fault), @@ -867,4 +891,10 @@ registered DATETIME, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, note VARCHAR(255), PRIMARY KEY(warp_id), @@ -923,4 +953,10 @@ fault SMALLINT, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, PRIMARY KEY(warp_id, skycell_id, tess_id), KEY(good_frac), @@ -948,4 +984,5 @@ tess_id VARCHAR(64), filter VARCHAR(64), + software_ver VARCHAR(16), note VARCHAR(255), PRIMARY KEY(stack_id), @@ -995,4 +1032,5 @@ quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT, + software_ver VARCHAR(16), PRIMARY KEY(stack_id), KEY(dtime_stack), @@ -1017,4 +1055,10 @@ exposure TINYINT DEFAULT 0, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, diff_mode SMALLINT NOT NULL, note VARCHAR(255), @@ -1080,4 +1124,10 @@ fault SMALLINT, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, PRIMARY KEY(diff_id, skycell_id), KEY(good_frac), Index: trunk/ippTools/share/warptool_coalesce_run.sql =================================================================== --- trunk/ippTools/share/warptool_coalesce_run.sql (revision 28043) +++ trunk/ippTools/share/warptool_coalesce_run.sql (revision 28043) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from warpSkyfile where warp_id = %lld Index: trunk/ippTools/src/camtool.c =================================================================== --- trunk/ippTools/src/camtool.c (revision 28041) +++ trunk/ippTools/src/camtool.c (revision 28043) @@ -481,4 +481,40 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_ppimage, config->args, "-ver_ppimage", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_psastro) { + software_ver = pxMergeCodeVersions(software_ver,ver_psastro); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + if (ver_ppimage) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppimage); + } + if (ver_streaks) { + software_ver = pxMergeCodeVersions(software_ver,ver_streaks); + } // Get this from the chipRun @@ -595,5 +631,11 @@ path_base, fault, - quality + software_ver, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory, + quality ); @@ -620,4 +662,29 @@ // else continue on... + + // Set camRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "camRun", "cam_id", cam_id, software_ver)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set camRun.software_ver for cam_id: %" PRId64, + cam_id); + psFree(output); + return(false); + } + // Set chipRun.maskfrac* to the appropriate values. + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "camRun", "cam_id",cam_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set camRun.maskstats for cam_id: %" PRId64, + cam_id); + psFree(output); + return(false); + } + } + // since there is only one exp per 'new' set camRun.state = 'full' // propagate magicked state from chipRun @@ -628,4 +695,5 @@ return false; } + psFree(row); Index: trunk/ippTools/src/camtoolConfig.c =================================================================== --- trunk/ippTools/src/camtoolConfig.c (revision 28041) +++ trunk/ippTools/src/camtoolConfig.c (revision 28043) @@ -174,4 +174,19 @@ psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_ppimage", 0, "define ppImage version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -processedexp psMetadata *processedexpArgs = psMetadataAlloc(); Index: trunk/ippTools/src/chiptool.c =================================================================== --- trunk/ippTools/src/chiptool.c (revision 28041) +++ trunk/ippTools/src/chiptool.c (revision 28043) @@ -585,4 +585,40 @@ PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_ppimage, config->args, "-ver_ppimage", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_psastro) { + ver_code = pxMergeCodeVersions(ver_code,ver_psastro); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_ppimage) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppimage); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } + // default values PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -661,5 +697,11 @@ fault, quality, - magicked + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { // rollback @@ -1353,4 +1395,32 @@ psMetadata *row = output->data[i]; + psS64 chip_id = psMetadataLookupS64(NULL,row,"chip_id"); + + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"chiptool_coalesce_run.sql",chip_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + psMetadataAddStr(row, PS_LIST_TAIL, "software_ver", 0, "Software version", software_ver); + psMetadataAddS32(row, PS_LIST_TAIL, "maskfrac_npix", 0, "Number of pixels used for mask stats", maskfrac_npix); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_static", 0, "Fraction of static masked pixels", maskfrac_static); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_dynamic", 0, "Fraction of dynamic masked pixels", maskfrac_dynamic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_magic", 0, "Fraction of magic masked pixels", maskfrac_magic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_advisory", 0, "Fraction of advisory pixels", maskfrac_advisory); + psWarning("ADVANCE %ld %s %d %f %f %f %f\n",chip_id,software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory); chipRunRow *chipRun = chipRunObjectFromMetadata(row); if (!psDBTransaction(config->dbh)) { @@ -1359,4 +1429,25 @@ } + // Set chipRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "chipRun", "chip_id", chip_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set chipRun.maskfrac* to the appropriate values. + if (!pxSetRunMaskfrac(config, "chipRun", "chip_id",chip_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } // set chipRun.state to 'stop' and update the magicked state if (!pxchipRunSetState(config, chipRun->chip_id, "full", chipRun->magicked)) { @@ -1383,16 +1474,16 @@ // stop if (!pxcamQueueByChipID(config, - chipRun->chip_id, - chipRun->workdir, - chipRun->label, - chipRun->data_group, - chipRun->dist_group, - chipRun->reduction, - chipRun->expgroup, - chipRun->dvodb, - chipRun->tess_id, - chipRun->end_stage, - chipRun->magicked, - NULL // note does not propagate + chipRun->chip_id, + chipRun->workdir, + chipRun->label, + chipRun->data_group, + chipRun->dist_group, + chipRun->reduction, + chipRun->expgroup, + chipRun->dvodb, + chipRun->tess_id, + chipRun->end_stage, + chipRun->magicked, + NULL // note does not propagate )) { if (!psDBRollback(config->dbh)) { Index: trunk/ippTools/src/chiptoolConfig.c =================================================================== --- trunk/ippTools/src/chiptoolConfig.c (revision 28041) +++ trunk/ippTools/src/chiptoolConfig.c (revision 28043) @@ -181,4 +181,18 @@ psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", 0); + + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_ppimage", 0, "define ppImage version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); // -processedimfile Index: trunk/ippTools/src/difftool.c =================================================================== --- trunk/ippTools/src/difftool.c (revision 28041) +++ trunk/ippTools/src/difftool.c (revision 28043) @@ -162,4 +162,10 @@ exposure, false, + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory diff_mode, note @@ -549,4 +555,36 @@ PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_ppsub, config->args, "-ver_ppsub", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s ppsub %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_ppsub,ver_streaks); + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_ppsub) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppsub); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } if (!psDBTransaction(config->dbh)) { @@ -582,5 +620,11 @@ fault, quality, - magicked + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { if (!psDBRollback(config->dbh)) { @@ -671,4 +715,48 @@ psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"difftool_coalesce_run.sql",diff_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set diffRun.software_ver to the appropriate value + if (software_ver) { + if (!pxSetRunSoftware(config, "diffRun", "diff_id", diff_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set diffRun.software_ver for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + // Set diffRun.maskfrac* to the appropriate values. + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "diffRun", "diff_id", diff_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set diffRun.software_ver for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } // set diffRun.state to 'full' if (!setdiffRunState(config, diff_id, "full", magicked)) { @@ -985,4 +1073,10 @@ false, 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory diff_mode, // diff_mode note @@ -1412,4 +1506,10 @@ true, // exposure 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_WARP_STACK, note @@ -1764,4 +1864,10 @@ true, // exposure false, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_WARP_WARP, note); // Run to insert @@ -2046,4 +2152,10 @@ false, // exposure 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_STACK_STACK, // diff_mode note Index: trunk/ippTools/src/difftoolConfig.c =================================================================== --- trunk/ippTools/src/difftoolConfig.c (revision 28041) +++ trunk/ippTools/src/difftoolConfig.c (revision 28043) @@ -126,4 +126,15 @@ psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_ppsub", 0, "define ppSub version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); // -advance Index: trunk/ippTools/src/pxcam.c =================================================================== --- trunk/ippTools/src/pxcam.c (revision 28041) +++ trunk/ippTools/src/pxcam.c (revision 28043) @@ -152,5 +152,5 @@ char *tess_id, char *end_stage, - psS64 magicked, + psS64 magicked, char *note) { @@ -184,4 +184,10 @@ end_stage ? end_stage : "NULL", (long long) magicked, + "NULL", // This shouldn't be propagated + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", note ? note : "NULL", (long long)chip_id Index: trunk/ippTools/src/pxchip.c =================================================================== --- trunk/ippTools/src/pxchip.c (revision 28041) +++ trunk/ippTools/src/pxchip.c (revision 28043) @@ -277,4 +277,10 @@ end_stage, 0, // magicked + NULL, + 0, + NAN, + NAN, + NAN, + NAN, note ) Index: trunk/ippTools/src/pxtools.c =================================================================== --- trunk/ippTools/src/pxtools.c (revision 28041) +++ trunk/ippTools/src/pxtools.c (revision 28043) @@ -46,4 +46,119 @@ if (!strcmp(state, "scrubbed")) return true; return false; +} + +psString pxMergeCodeVersions(psString version1, psString version2) { + psString out = NULL; + + bool mod1 = false; + bool mod2 = false; + + psS32 num1; + psS32 num2; + psS32 numO; + + if (!version1) { + psStringAppend(&out,version2); + return(out); + } + if (!version2) { + psStringAppend(&out,version1); + return(out); + } + + if (strchr(version1,'M')) { + psStringSubstitute(&version1,"M",""); + mod1 = true; + } + if (strchr(version2,'M')) { + psStringSubstitute(&version2,"M",""); + mod2 = true; + } + + num1 = strtol(version1,NULL,10); + num2 = strtol(version2,NULL,10); + + if (num1 >= num2) { + numO = num1; + } + else { + numO = num2; + } + + psStringAppend(&out,"%" PRId32,numO); + if (mod1 || mod2) { + psStringAppend(&out,"M"); + } + return(out); +} + +bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, + psS32 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psF32 *maskfrac_magic, psF32 *maskfrac_advisory) { + psString query = pxDataGet(dbQFile); +/* psString text_id = NULL; */ +/* psStringAppend(&text_id," %" PRId64,stage_id); */ +/* psStringSubstitute(&query,text_id,"@STAGE_ID@"); */ +/* psFree(text_id); */ + if (!p_psDBRunQueryF(config->dbh, query, stage_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS32 this_npix = psMetadataLookupS32(NULL, row, "maskfrac_npix"); + psF32 this_static = psMetadataLookupF32(NULL, row, "maskfrac_static"); + psF32 this_dynamic = psMetadataLookupF32(NULL, row, "maskfrac_dynamic"); + psF32 this_magic = psMetadataLookupF32(NULL, row, "maskfrac_magic"); + psF32 this_advisory = psMetadataLookupF32(NULL, row, "maskfrac_advisory"); + psString this_version = psMetadataLookupStr(NULL, row, "software_ver"); + + *software_ver = pxMergeCodeVersions(*software_ver,this_version); + + *maskfrac_static = ((*maskfrac_static * *maskfrac_npix) + (this_npix * this_static)) / (this_npix + *maskfrac_npix); + *maskfrac_dynamic = ((*maskfrac_dynamic * *maskfrac_npix) + (this_npix * this_dynamic)) / (this_npix + *maskfrac_npix); + *maskfrac_magic = ((*maskfrac_magic * *maskfrac_npix) + (this_npix * this_magic)) / (this_npix + *maskfrac_npix); + *maskfrac_advisory = ((*maskfrac_advisory * *maskfrac_npix) + (this_npix * this_advisory)) / (this_npix + *maskfrac_npix); + *maskfrac_npix += this_npix; + } + psFree(output); + return(true); +} + +bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psString software_ver) { + char *query = "UPDATE %s SET software_ver = '%s' WHERE %s = %" PRId64; + printf(query,tableName,software_ver,stage_id_name,stage_id); + if (!p_psDBRunQueryF(config->dbh,query,tableName,software_ver,stage_id_name,stage_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set software version for %s %" PRId64,stage_id_name,stage_id); + return(false); + } + + return(true); +} +bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psS32 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psF32 maskfrac_magic, psF32 maskfrac_advisory) { + char *query = "UPDATE %s SET maskfrac_npix = %d, maskfrac_static = %f, maskfrac_dynamic = %f, maskfrac_magic = %f, maskfrac_advisory = %f WHERE %s = %" PRId64; + if (!p_psDBRunQueryF(config->dbh,query,tableName,maskfrac_npix,maskfrac_static, + maskfrac_dynamic, maskfrac_magic,maskfrac_advisory,stage_id_name,stage_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set maskfrac stats for %s %" PRId64,stage_id_name,stage_id); + return(false); + } + + + + return(true); } Index: trunk/ippTools/src/pxtools.h =================================================================== --- trunk/ippTools/src/pxtools.h (revision 28041) +++ trunk/ippTools/src/pxtools.h (revision 28043) @@ -53,4 +53,14 @@ bool pxIsValidState(const char *state); +psString pxMergeCodeVersions(psString version1, psString version2); +bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, + psS32 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psF32 *maskfrac_magic, psF32 *maskfrac_advisory); +bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psString software_ver); +bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psS32 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psF32 maskfrac_magic, psF32 maskfrac_advisory); + bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows); bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *field, char *name, char *op); Index: trunk/ippTools/src/pxwarp.c =================================================================== --- trunk/ippTools/src/pxwarp.c (revision 28041) +++ trunk/ippTools/src/pxwarp.c (revision 28043) @@ -60,4 +60,10 @@ NULL, // registered 0, // magicked set to zero when created may get updated when warpRun goes to 'full' + NULL, // version + 0, // maskfrac_npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory NULL // note )) { Index: trunk/ippTools/src/stacktool.c =================================================================== --- trunk/ippTools/src/stacktool.c (revision 28041) +++ trunk/ippTools/src/stacktool.c (revision 28043) @@ -370,4 +370,5 @@ tess_id, filter, + NULL, // software_ver note); @@ -526,4 +527,5 @@ tess_id, filter, + NULL, // software_ver note); @@ -844,4 +846,30 @@ PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_ppstack, config->args, "-ver_ppstack", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s ppstack %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_ppstack,ver_streaks); + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + if (ver_ppstack) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstack); + } + if (ver_streaks) { + software_ver = pxMergeCodeVersions(software_ver,ver_streaks); + } + // default values PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -883,4 +911,5 @@ good_frac, fault, + software_ver, quality )) { @@ -893,4 +922,14 @@ if (fault == 0) { + // Set stackRun software if we are finished. + if (!pxSetRunSoftware(config, "stackRun", "stack_id", stack_id, software_ver)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set stackRun.software_ver for stack_id: %" PRId64, + stack_id); + return(false); + } + if (!setstackRunState(config, stack_id, "full")) { if (!psDBRollback(config->dbh)) { Index: trunk/ippTools/src/stacktoolConfig.c =================================================================== --- trunk/ippTools/src/stacktoolConfig.c (revision 28041) +++ trunk/ippTools/src/stacktoolConfig.c (revision 28043) @@ -171,4 +171,11 @@ psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_ppstack", 0, "define ppStack version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + // -sumskyfile psMetadata *sumskyfileArgs= psMetadataAlloc(); Index: trunk/ippTools/src/warptool.c =================================================================== --- trunk/ippTools/src/warptool.c (revision 28041) +++ trunk/ippTools/src/warptool.c (revision 28043) @@ -177,5 +177,11 @@ registered, 0, // magicked - note + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory + note ); if (!warpRun) { @@ -1019,8 +1025,42 @@ PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_pswarp, config->args, "-ver_pswarp", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s pswarp %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_pswarp,ver_streaks); + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_pswarp) { + ver_code = pxMergeCodeVersions(ver_code,ver_pswarp); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } + // default values PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + // we don't want to insert the last skyfile in a run but then not mark the // run as 'stop' @@ -1050,5 +1090,11 @@ fault, quality, - magicked + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { if (!psDBRollback(config->dbh)) { @@ -1115,4 +1161,47 @@ bool status; psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); + + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"warptool_coalesce_run.sql",warp_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set warpRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "warpRun", "warp_id", warp_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, + warp_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set warpRun.maskfrac* to the appropriate values. + if (!pxSetRunMaskfrac(config, "warpRun", "warp_id", warp_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, + warp_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + + if (!status) { psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id"); Index: trunk/ippTools/src/warptoolConfig.c =================================================================== --- trunk/ippTools/src/warptoolConfig.c (revision 28041) +++ trunk/ippTools/src/warptoolConfig.c (revision 28043) @@ -224,4 +224,18 @@ psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_pswarp", 0, "define pswarp version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -warped psMetadata *warpedArgs = psMetadataAlloc(); Index: trunk/ippconfig/gpc1/camera.config =================================================================== --- trunk/ippconfig/gpc1/camera.config (revision 28041) +++ trunk/ippconfig/gpc1/camera.config (revision 28043) @@ -150,4 +150,7 @@ PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.NAME} # Rule for generating photcode -BURNTOOL.STATE.GOOD S16 14 # Value for burntool_state with the most recent bt version. -BURNTOOL.STATE.GOOD.UPDATE S16 13 # for upddate processing accept earlier version +BURNTOOL.STATE.GOOD S16 14 # Value for burntool_state with the most recent bt version. +BURNTOOL.STATE.GOOD.UPDATE S16 13 # for upddate processing accept earlier version + +FOV F32 15000 # Field of view of unvignetted region in FPA pixels. +NPIX_INTERCHIP S32 0 Index: trunk/ippconfig/recipes/ppImage.config =================================================================== --- trunk/ippconfig/recipes/ppImage.config (revision 28041) +++ trunk/ippconfig/recipes/ppImage.config (revision 28043) @@ -35,5 +35,5 @@ USE.BEST.BURNTOOL BOOL FALSE # require the best burntooled data TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool - +MASK.STATS BOOL FALSE # calculate mask statistics. GAIN.OVERRIDE BOOL TRUE # Override a non-finite gain? @@ -67,4 +67,9 @@ # Which regions identified by burntool to mask. 0x01 : only unfit trails, 0x02 "up" trails, 0x04 "down" trails BURNTOOL.TRAILS U16 0x07 + +MASKSTAT.STATIC U32 0x01f +MASKSTAT.DYNAMIC U32 0x060 +MASKSTAT.MAGIC U32 0x800 +MASKSTAT.ADVISORY U32 0x080 # Non-linearity correction @@ -169,4 +174,5 @@ ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? BACKGROUND BOOL TRUE # Subtract background? + MASK.STATS BOOL TRUE # Calculate Mask statistics. END @@ -1822,4 +1828,5 @@ ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? BACKGROUND BOOL FALSE # Subtract background? + MASK.STATS BOOL TRUE # Calculate Mask statistics END @@ -1846,4 +1853,5 @@ ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? BACKGROUND BOOL TRUE # Subtract background? + MASK.STATS BOOL TRUE # Calculate Mask statistics END @@ -1870,4 +1878,5 @@ ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? BACKGROUND BOOL TRUE # Subtract background? + MASK.STATS BOOL TRUE # Calculate Mask statistics PATTERN.ROW BOOL TRUE # Subtract pattern noise PATTERN.CELL BOOL FALSE # Subtract pattern noise Index: trunk/ippconfig/recipes/ppStats.config =================================================================== --- trunk/ippconfig/recipes/ppStats.config (revision 28041) +++ trunk/ippconfig/recipes/ppStats.config (revision 28043) @@ -85,4 +85,12 @@ HEADER STR IQ_M4_UQ + HEADER STR PSLIB_V + HEADER STR MODULE_V + HEADER STR PHOT_V + HEADER STR ASTRO_V + HEADER STR STATS_V + HEADER STR IMAGE_V + HEADER STR STREAK_V + HEADER STR DETREND.MASK DETREND.BIAS DETREND.DARK DETREND.SHUTTER DETREND.FLAT DETREND.FRINGE @@ -91,4 +99,11 @@ ANALYSIS STR PSASTRO.HEADER ANALYSIS STR DETREND + + SUMMARY STR MASKFRAC_STATIC + SUMMARY STR MASKFRAC_MAGIC + SUMMARY STR MASKFRAC_ADVISORY + SUMMARY STR MASKFRAC_DYNAMIC + SUMMARY STR MASKFRAC_NPIX + END @@ -103,7 +118,21 @@ HEADER STR ZPT_ERR + HEADER STR PSLIB_V + HEADER STR MODULE_V + HEADER STR PHOT_V + HEADER STR ASTRO_V + HEADER STR STATS_V + HEADER STR IMAGE_V + HEADER STR STREAK_V + ANALYSIS MULTI # metadata blocks to search in chip/cell/readout->analysis ANALYSIS STR PSPHOT.HEADER ANALYSIS STR PSASTRO.HEADER + + ANALYSIS STR MASKFRAC_STATIC + ANALYSIS STR MASKFRAC_MAGIC + ANALYSIS STR MASKFRAC_ADVISORY + ANALYSIS STR MASKFRAC_DYNAMIC + ANALYSIS STR MASKFRAC_NPIX END @@ -241,7 +270,75 @@ SUMMARY MULTI + HEADER STR PSLIB_V + HEADER STR MODULE_V + HEADER STR PHOT_V + HEADER STR STATS_V + HEADER STR WARP_V + HEADER STR STREAK_V + STAT STR ROBUST_MEDIAN STAT STR ROBUST_STDEV # Background statistics estimators SUMMARY STR GOOD_PIXEL_NUM # Number of good pixels SUMMARY STR GOOD_PIXEL_FRAC # Fraction of good pixels -END + + ANALYSIS MULTI + ANALYSIS STR MASKFRAC_STATIC + ANALYSIS STR MASKFRAC_MAGIC + ANALYSIS STR MASKFRAC_ADVISORY + ANALYSIS STR MASKFRAC_DYNAMIC + ANALYSIS STR MASKFRAC_NPIX +END + +# Based on WARPSTATS +# basic stats for stacked images +STACKSTATS METADATA + HEADER MULTI + CONCEPT MULTI + STAT MULTI + SUMMARY MULTI + + HEADER STR PSLIB_V + HEADER STR MODULE_V + HEADER STR PHOT_V + HEADER STR STATS_V + HEADER STR STACK_V + + STAT STR ROBUST_MEDIAN + STAT STR ROBUST_STDEV # Background statistics estimators + SUMMARY STR GOOD_PIXEL_NUM # Number of good pixels + SUMMARY STR GOOD_PIXEL_FRAC # Fraction of good pixels + + ANALYSIS MULTI + ANALYSIS STR MASKFRAC_STATIC + ANALYSIS STR MASKFRAC_MAGIC + ANALYSIS STR MASKFRAC_ADVISORY + ANALYSIS STR MASKFRAC_DYNAMIC + ANALYSIS STR MASKFRAC_NPIX +END + +# basic stats for diffed images +DIFFSTATS METADATA + HEADER MULTI + CONCEPT MULTI + STAT MULTI + SUMMARY MULTI + + HEADER STR PSLIB_V + HEADER STR MODULE_V + HEADER STR PHOT_V + HEADER STR STATS_V + HEADER STR PPSUB_V + HEADER STR STREAK_V + + STAT STR ROBUST_MEDIAN + STAT STR ROBUST_STDEV # Background statistics estimators + SUMMARY STR GOOD_PIXEL_NUM # Number of good pixels + SUMMARY STR GOOD_PIXEL_FRAC # Fraction of good pixels + + ANALYSIS MULTI + ANALYSIS STR MASKFRAC_STATIC + ANALYSIS STR MASKFRAC_MAGIC + ANALYSIS STR MASKFRAC_ADVISORY + ANALYSIS STR MASKFRAC_DYNAMIC + ANALYSIS STR MASKFRAC_NPIX +END Index: trunk/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- trunk/ippconfig/recipes/ppStatsFromMetadata.config (revision 28041) +++ trunk/ippconfig/recipes/ppStatsFromMetadata.config (revision 28043) @@ -171,4 +171,20 @@ ENTRY VAL IQ_M2S_U F32 SAMPLE_MEAN -iq_m2s_uq ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL PHOT_V STR CONSTANT -ver_psphot + ENTRY VAL ASTRO_V STR CONSTANT -ver_psastro + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats + ENTRY VAL IMAGE_V STR CONSTANT -ver_ppimage + ENTRY VAL STREAK_V STR CONSTANT -ver_streaks + + # Mask stats values + ENTRY VAL MASKFRAC_NPIX S32 CONSTANT -maskfrac_npix + ENTRY VAL MASKFRAC_STATIC F32 CONSTANT -maskfrac_static + ENTRY VAL MASKFRAC_DYNAMIC F32 CONSTANT -maskfrac_dynamic + ENTRY VAL MASKFRAC_MAGIC F32 CONSTANT -maskfrac_magic + ENTRY VAL MASKFRAC_ADVISORY F32 CONSTANT -maskfrac_advisory END @@ -230,4 +246,13 @@ ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL PHOT_V STR CONSTANT -ver_psphot + ENTRY VAL ASTRO_V STR CONSTANT -ver_psastro + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats + ENTRY VAL IMAGE_V STR CONSTANT -ver_ppimage + ENTRY VAL STREAK_V STR CONSTANT -ver_streaks END @@ -246,4 +271,20 @@ ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL PHOT_V STR CONSTANT -ver_psphot + ENTRY VAL ASTRO_V STR CONSTANT -ver_psastro + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats + ENTRY VAL IMAGE_V STR CONSTANT -ver_ppimage + ENTRY VAL STREAK_V STR CONSTANT -ver_streaks + + # Mask stats values + ENTRY VAL MASKFRAC_NPIX S32 CONSTANT -maskfrac_npix + ENTRY VAL MASKFRAC_STATIC F32 CONSTANT -maskfrac_static + ENTRY VAL MASKFRAC_DYNAMIC F32 CONSTANT -maskfrac_dynamic + ENTRY VAL MASKFRAC_MAGIC F32 CONSTANT -maskfrac_magic + ENTRY VAL MASKFRAC_ADVISORY F32 CONSTANT -maskfrac_advisory END @@ -261,4 +302,20 @@ ENTRY VAL ACCEPT BOOL CONSTANT -accept ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL PHOT_V STR CONSTANT -ver_psphot + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats + ENTRY VAL WARP_V STR CONSTANT -ver_pswarp + ENTRY VAL STREAK_V STR CONSTANT -ver_streaks + + # Mask stats values + ENTRY VAL MASKFRAC_NPIX S32 CONSTANT -maskfrac_npix + ENTRY VAL MASKFRAC_STATIC F32 CONSTANT -maskfrac_static + ENTRY VAL MASKFRAC_DYNAMIC F32 CONSTANT -maskfrac_dynamic + ENTRY VAL MASKFRAC_MAGIC F32 CONSTANT -maskfrac_magic + ENTRY VAL MASKFRAC_ADVISORY F32 CONSTANT -maskfrac_advisory + END @@ -288,4 +345,12 @@ ENTRY VAL GOOD_PIXEL_FRAC F32 MEAN -good_frac ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL PHOT_V STR CONSTANT -ver_psphot + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats + ENTRY VAL STACK_V STR CONSTANT -ver_ppstack + ENTRY VAL STREAK_V STR CONSTANT -ver_streaks END @@ -312,4 +377,19 @@ ENTRY VAL GOOD_PIXEL_FRAC F32 MEAN -good_frac ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL PHOT_V STR CONSTANT -ver_psphot + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats + ENTRY VAL PPSUB_V STR CONSTANT -ver_ppsub + ENTRY VAL STREAK_V STR CONSTANT -ver_streaks + + # Mask stats values + ENTRY VAL MASKFRAC_NPIX S32 CONSTANT -maskfrac_npix + ENTRY VAL MASKFRAC_STATIC F32 CONSTANT -maskfrac_static + ENTRY VAL MASKFRAC_DYNAMIC F32 CONSTANT -maskfrac_dynamic + ENTRY VAL MASKFRAC_MAGIC F32 CONSTANT -maskfrac_magic + ENTRY VAL MASKFRAC_ADVISORY F32 CONSTANT -maskfrac_advisory END Index: trunk/ippconfig/recipes/ppSub.config =================================================================== --- trunk/ippconfig/recipes/ppSub.config (revision 28041) +++ trunk/ippconfig/recipes/ppSub.config (revision 28043) @@ -21,4 +21,11 @@ MASK.BAD STR CONV.BAD # Mask value to give bad pixels MASK.POOR STR CONV.POOR # Mask value to give poor pixels + +MASK.STATS BOOL TRUE # calculate mask statistics. +MASKSTAT.STATIC U32 0x01f +MASKSTAT.DYNAMIC U32 0x060 +MASKSTAT.MAGIC U32 0x800 +MASKSTAT.ADVISORY U32 0x080 + LOW.THRESHOLD F32 4.0 # Mask pixels below this threshold (std dev) Index: trunk/ippconfig/recipes/psastro.config =================================================================== --- trunk/ippconfig/recipes/psastro.config (revision 28041) +++ trunk/ippconfig/recipes/psastro.config (revision 28043) @@ -163,4 +163,12 @@ EXTRACT_MAX_MAG F32 -15.0 +MASK.STATS BOOL TRUE +MASKSTAT.STATIC U32 0x01f +MASKSTAT.DYNAMIC U32 0x060 +MASKSTAT.MAGIC U32 0x800 +MASKSTAT.ADVISORY U32 0x080 + + + # 2MASS default configuration (use 2MASS_J as ref magnitude) PSASTRO.CATDIR STR 2MASS Index: trunk/ippconfig/recipes/pswarp.config =================================================================== --- trunk/ippconfig/recipes/pswarp.config (revision 28041) +++ trunk/ippconfig/recipes/pswarp.config (revision 28043) @@ -12,4 +12,11 @@ PSF BOOL TRUE # Measure PSF for warped image? +MASK.STATS BOOL TRUE # calculate mask statistics. +MASKSTAT.STATIC U32 0x01f +MASKSTAT.DYNAMIC U32 0x060 +MASKSTAT.MAGIC U32 0x800 +MASKSTAT.ADVISORY U32 0x080 + + # Default recipe for warping WARP METADATA Index: trunk/magic/censorObjects/src/censorVersion.c =================================================================== --- trunk/magic/censorObjects/src/censorVersion.c (revision 28041) +++ trunk/magic/censorObjects/src/censorVersion.c (revision 28043) @@ -58,4 +58,6 @@ psString source = censorSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "CENSOR_V", 0, NULL, source); + psStringPrepend(&version, "censor version: "); psStringPrepend(&source, "censor source: "); Index: trunk/magic/remove/src/streaksVersion.c =================================================================== --- trunk/magic/remove/src/streaksVersion.c (revision 28041) +++ trunk/magic/remove/src/streaksVersion.c (revision 28043) @@ -58,4 +58,6 @@ psString source = streaksSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "STREAK_V", 0, NULL, source); + psStringPrepend(&version, "%s version: ", streaksProgram); psStringPrepend(&source, "%s source: ", streaksProgram); Index: trunk/ppArith/src/ppArithVersion.c =================================================================== --- trunk/ppArith/src/ppArithVersion.c (revision 28041) +++ trunk/ppArith/src/ppArithVersion.c (revision 28043) @@ -83,4 +83,6 @@ psString source = ppArithSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "ARITH_V", PS_META_REPLACE, NULL, PPARITH_VERSION); + psStringPrepend(&version, "ppArith version: "); psStringPrepend(&version, "ppArith source: "); Index: trunk/ppImage/src/Makefile.am =================================================================== --- trunk/ppImage/src/Makefile.am (revision 28041) +++ trunk/ppImage/src/Makefile.am (revision 28043) @@ -44,4 +44,5 @@ ppImageRebinReadout.c \ ppImageMosaic.c \ + ppImageMaskStats.c \ ppImagePhotom.c \ ppImageAstrom.c \ Index: trunk/ppImage/src/ppImage.h =================================================================== --- trunk/ppImage/src/ppImage.h (revision 28041) +++ trunk/ppImage/src/ppImage.h (revision 28043) @@ -49,4 +49,6 @@ bool applyParity; // Apply Cell parities + bool doMaskStats; // Calculate mask statistics + bool doCrosstalkMeasure; // measure crosstalk signal bool doCrosstalkCorrect; // apply crosstalk correction @@ -110,4 +112,10 @@ char *normClass; // class to use for per-class normalization + + psU16 maskstat_static; + psU16 maskstat_dynamic; + psU16 maskstat_magic; + psU16 maskstat_advisory; + } ppImageOptions; @@ -268,4 +276,7 @@ ); +// Calculate Mask statistics +bool ppImageMaskStats(pmConfig *config, pmFPAview *view, psMetadata *stats); + // calculate stats from headers and concepts bool ppImageMetadataStats(pmConfig *config, // Configuration Index: trunk/ppImage/src/ppImageLoop.c =================================================================== --- trunk/ppImage/src/ppImageLoop.c (revision 28041) +++ trunk/ppImage/src/ppImageLoop.c (revision 28043) @@ -196,4 +196,11 @@ } + if (options->doMaskStats) { + //if (!ppImageMaskStats(config, view, options)) { + if (!ppImageMaskStats(config, view, stats)) { + ESCAPE("Unable to do Mask stats"); + } + } + // these may be used by ppImageSubtractBackground. // if these are defined as internal files, drop them here @@ -252,5 +259,5 @@ ppImageFileCheck(config); } - + // Calculate summary statistics from FPA Metadata if (!ppImageMetadataStats(config, stats, options)) { Index: trunk/ppImage/src/ppImageMaskStats.c =================================================================== --- trunk/ppImage/src/ppImageMaskStats.c (revision 28043) +++ trunk/ppImage/src/ppImageMaskStats.c (revision 28043) @@ -0,0 +1,87 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + + +bool ppImageMaskStats(pmConfig *config, pmFPAview *view, psMetadata *stats) { + PS_ASSERT_PTR_NON_NULL(view, false); + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status; + + pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.CHIP"); + if (!status) { + psError(PS_ERR_UNEXPECTED_NULL, true, "PPIMAGE.CHIP file is not defined"); + return(false); + } + + psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, "PPIMAGE"); + + psU16 staticMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.STATIC"); + psU16 magicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.MAGIC"); + psU16 dynamicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.DYNAMIC"); + psU16 advisoryMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.ADVISORY"); + + psS32 Npix_valid = 0; + psS32 Npix_static = 0; + psS32 Npix_magic = 0; + psS32 Npix_dynamic = 0; + psS32 Npix_advisory = 0; + + pmChip *chip = pmFPAviewThisChip(view, input->fpa); // Chip of interest + if (!chip) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find chip"); + return(false); + } + if (chip->cells->n == 0) { + psWarning("Chip has no cells"); + return(true); + } + if (chip->cells->n > 1) { + psWarning("Chip has %ld cells; only the first will be processed", chip->cells->n); + } + pmCell *cell = chip->cells->data[0]; // Cell of interest + if (!cell || !cell->process || !cell->file_exists) { + // Nothing to process + return(true); + } + if (cell->readouts->n == 0) { + psWarning("Cell has no readouts"); + return(true); + } + if (cell->readouts->n > 1) { + psWarning("Cell has %ld readouts; only the first will be processed", cell->readouts->n); + } + pmReadout *readout = cell->readouts->data[0]; // Readout of interest; + if (!readout || !readout->data_exists) { + // Nothing to process + return(true); + } + + psImage *mask = readout->mask; // Mask of interest; + psWarning("In ppImageMaskStats: %d %ld\n",Npix_valid, (long) mask); + + if (!pmSingleImageMaskStats(mask,&Npix_valid,&Npix_static,&Npix_magic, + &Npix_dynamic,&Npix_advisory, + staticMaskVal,magicMaskVal, + dynamicMaskVal,advisoryMaskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate masks for readout."); + return(false); + } + psMetadataAddS32(stats, PS_LIST_TAIL,"MASKFRAC_NPIX", 0, + "Number of valid pixels", Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + psWarning("In ppImageMaskStats: %d %f %f %f %f\n",Npix_valid, (float) Npix_static / Npix_valid, + (float) Npix_dynamic / Npix_valid, (float) Npix_magic / Npix_valid, + (float) Npix_advisory / Npix_valid); + return(true); +} Index: trunk/ppImage/src/ppImageOptions.c =================================================================== --- trunk/ppImage/src/ppImageOptions.c (revision 28041) +++ trunk/ppImage/src/ppImageOptions.c (revision 28043) @@ -41,5 +41,6 @@ options->checkCTE = false; // Measure pixel-based variance options->applyParity = false; // Apply Cell parities - + options->doMaskStats = false; // Calculate mask fractions + // output files requested options->BaseFITS = false; // create output image @@ -243,4 +244,5 @@ options->doPatternRow = psMetadataLookupBool(NULL, recipe, "PATTERN.ROW"); options->doPatternCell = psMetadataLookupBool(NULL, recipe, "PATTERN.CELL"); + options->doMaskStats = psMetadataLookupBool(NULL, recipe, "MASK.STATS"); options->doStats = false; @@ -341,4 +343,10 @@ options->normClass = psMetadataLookupStr(NULL, recipe, "NORM.CLASS"); + options->maskstat_static = psMetadataLookupU16(NULL, recipe, "MASKSTAT.STATIC"); + options->maskstat_dynamic = psMetadataLookupU16(NULL, recipe, "MASKSTAT.DYNAMIC"); + options->maskstat_magic = psMetadataLookupU16(NULL, recipe, "MASKSTAT.MAGIC"); + options->maskstat_advisory = psMetadataLookupU16(NULL, recipe, "MASKSTAT.ADVISORY"); + + return options; } Index: trunk/ppImage/src/ppImageVersion.c =================================================================== --- trunk/ppImage/src/ppImageVersion.c (revision 28041) +++ trunk/ppImage/src/ppImageVersion.c (revision 28043) @@ -68,4 +68,6 @@ psString source = ppImageSource(); // ppImage software source + psMetadataAddStr(header, PS_LIST_TAIL, "IMAGE_V", PS_META_REPLACE, NULL, PPIMAGE_VERSION); + psStringPrepend(&version, "ppImage version: "); psStringPrepend(&source, "ppImage source: "); Index: trunk/ppMerge/src/ppMergeVersion.c =================================================================== --- trunk/ppMerge/src/ppMergeVersion.c (revision 28041) +++ trunk/ppMerge/src/ppMergeVersion.c (revision 28043) @@ -83,4 +83,6 @@ psString source = ppMergeSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "MERGE_V", PS_META_REPLACE, NULL, PPMERGE_VERSION); + psStringPrepend(&version, "ppMerge version: "); psStringPrepend(&source, "ppMerge source: "); Index: trunk/ppNoiseMap/src/ppNoiseMapVersion.c =================================================================== --- trunk/ppNoiseMap/src/ppNoiseMapVersion.c (revision 28041) +++ trunk/ppNoiseMap/src/ppNoiseMapVersion.c (revision 28043) @@ -61,4 +61,6 @@ psString source = ppNoiseMapSource(); // ppNoiseMap software source + psMetadataAddStr(header, PS_LIST_TAIL, "NOISEM_V", PS_META_REPLACE, NULL, PPNOISEMAP_VERSION); + psStringPrepend(&version, "ppNoiseMap version: "); psStringPrepend(&source, "ppNoiseMap source: "); Index: trunk/ppSkycell/src/ppSkycellVersion.c =================================================================== --- trunk/ppSkycell/src/ppSkycellVersion.c (revision 28041) +++ trunk/ppSkycell/src/ppSkycellVersion.c (revision 28043) @@ -81,4 +81,6 @@ psString source = ppSkycellSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "SKYCELL_V", PS_META_REPLACE, NULL, PPSKYCELL_VERSION); + psStringPrepend(&version, "ppSkycell version: "); psStringPrepend(&version, "ppSkycell source: "); @@ -89,5 +91,5 @@ psFree(version); psFree(source); - + return true; } Index: trunk/ppSmooth/src/ppSmoothVersion.c =================================================================== --- trunk/ppSmooth/src/ppSmoothVersion.c (revision 28041) +++ trunk/ppSmooth/src/ppSmoothVersion.c (revision 28043) @@ -75,4 +75,6 @@ psString source = ppSmoothSource(); // ppSmooth software source + psMetadataAddStr(header, PS_LIST_TAIL, "SMOOTH_V", PS_META_REPLACE, NULL, PPSMOOTH_VERSION); + psStringPrepend(&version, "ppSmooth version: "); psStringPrepend(&source, "ppSmooth source: "); @@ -83,5 +85,5 @@ psFree(version); psFree(source); - + return true; } Index: trunk/ppStack/src/ppStackVersion.c =================================================================== --- trunk/ppStack/src/ppStackVersion.c (revision 28041) +++ trunk/ppStack/src/ppStackVersion.c (revision 28043) @@ -74,4 +74,6 @@ psString source = ppStackSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "STACK_V", PS_META_REPLACE, NULL, PPSTACK_VERSION); + psStringPrepend(&version, "ppStack version: "); psStringPrepend(&source, "ppStack source: "); @@ -82,5 +84,5 @@ psFree(version); psFree(source); - + return true; } Index: trunk/ppStats/src/ppStatsVersion.c =================================================================== --- trunk/ppStats/src/ppStatsVersion.c (revision 28041) +++ trunk/ppStats/src/ppStatsVersion.c (revision 28043) @@ -49,4 +49,6 @@ psString source = ppStatsSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "STATS_V", PS_META_REPLACE, NULL, PPSTATS_VERSION); + psStringPrepend(&version, "ppStats version: "); psStringPrepend(&source, "ppStats source: "); Index: trunk/ppSub/src/Makefile.am =================================================================== --- trunk/ppSub/src/Makefile.am (revision 28041) +++ trunk/ppSub/src/Makefile.am (revision 28043) @@ -41,4 +41,5 @@ ppSubMakePSF.c \ ppSubMatchPSFs.c \ + ppSubMaskStats.c \ ppSubReadoutInverse.c \ ppSubReadoutJpeg.c \ Index: trunk/ppSub/src/ppSub.h =================================================================== --- trunk/ppSub/src/ppSub.h (revision 28041) +++ trunk/ppSub/src/ppSub.h (revision 28043) @@ -74,4 +74,7 @@ bool ppSubSetMasks(pmConfig *config ///< Configuration ); + +// Calculate mask statistics +bool ppSubMaskStats(pmConfig *config, pmFPAview *view, psMetadata *stats); /// Generate the PSF-matching kernel and convolve the images as needed. Most of this function involves Index: trunk/ppSub/src/ppSubLoop.c =================================================================== --- trunk/ppSub/src/ppSubLoop.c (revision 28041) +++ trunk/ppSub/src/ppSubLoop.c (revision 28043) @@ -144,5 +144,13 @@ return false; } - + // Do Mask Stats + { + pmFPAview *view = ppSubViewReadout(); // View to readout + if (!ppSubMaskStats(config, view,data->stats)) { + psError(psErrorCodeLast(), false, "Unable to generate mask statistics"); + return(false); + } + } + // generate the binned image used to write the jpeg if (!ppSubReadoutJpeg(config)) { Index: trunk/ppSub/src/ppSubMaskStats.c =================================================================== --- trunk/ppSub/src/ppSubMaskStats.c (revision 28043) +++ trunk/ppSub/src/ppSubMaskStats.c (revision 28043) @@ -0,0 +1,55 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppSub.h" + + +bool ppSubMaskStats(pmConfig *config, pmFPAview *view, psMetadata *stats) { + PS_ASSERT_PTR_NON_NULL(view, false); + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status; + + psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, PPSUB_RECIPE); + psU16 staticMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.STATIC"); + psU16 magicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.MAGIC"); + psU16 dynamicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.DYNAMIC"); + psU16 advisoryMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.ADVISORY"); + + psS32 Npix_valid = 0; + psS32 Npix_static = 0; + psS32 Npix_magic = 0; + psS32 Npix_dynamic = 0; + psS32 Npix_advisory = 0; + + psWarning("ppSubMaskStats: %d %d %d %d\n",staticMaskVal,magicMaskVal,dynamicMaskVal,advisoryMaskVal); + + pmReadout *readout = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); + if (!readout || !readout->data_exists) { + // Nothing to process + return(true); + } + psImage *mask = readout->mask; // Mask of interest; + if (!pmSingleImageMaskStats(mask,&Npix_valid,&Npix_static,&Npix_magic, + &Npix_dynamic,&Npix_advisory, + staticMaskVal,magicMaskVal, + dynamicMaskVal,advisoryMaskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate masks for readout."); + return(false); + } + psMetadataAddS32(stats, PS_LIST_TAIL,"MASKFRAC_NPIX", 0, + "Number of valid pixels", Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + return(true); +} Index: trunk/ppSub/src/ppSubVersion.c =================================================================== --- trunk/ppSub/src/ppSubVersion.c (revision 28041) +++ trunk/ppSub/src/ppSubVersion.c (revision 28043) @@ -86,4 +86,6 @@ psString source = ppSubSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "PPSUB_V", PS_META_REPLACE, NULL, PPSUB_VERSION); + psStringPrepend(&version, "ppSub version: "); psStringPrepend(&source, "ppSub source: "); Index: trunk/ppTranslate/src/ppTranslateVersion.c =================================================================== --- trunk/ppTranslate/src/ppTranslateVersion.c (revision 28041) +++ trunk/ppTranslate/src/ppTranslateVersion.c (revision 28043) @@ -67,4 +67,6 @@ psString source = ppTranslateSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "TRANSL_V", PS_META_REPLACE, NULL, PPTRANSLATE_VERSION); + psStringPrepend(&version, "ppTranslate version: "); psStringPrepend(&source, "ppTranslate source: "); Index: trunk/ppViz/src/ppCoord/ppCoordVersion.c =================================================================== --- trunk/ppViz/src/ppCoord/ppCoordVersion.c (revision 28041) +++ trunk/ppViz/src/ppCoord/ppCoordVersion.c (revision 28043) @@ -81,4 +81,6 @@ psString source = ppCoordSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "COORD_V", PS_META_REPLACE, NULL, PPCOORD_VERSION); + psStringPrepend(&version, "ppCoord version: "); psStringPrepend(&version, "ppCoord source: "); Index: trunk/ppViz/src/ppVizPSF/ppVizPSFVersion.c =================================================================== --- trunk/ppViz/src/ppVizPSF/ppVizPSFVersion.c (revision 28041) +++ trunk/ppViz/src/ppVizPSF/ppVizPSFVersion.c (revision 28043) @@ -81,4 +81,6 @@ psString source = ppVizPSFSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "VIZPSF_V", PS_META_REPLACE, NULL, PPVIZPSF_VERSION); + psStringPrepend(&version, "ppVizPSF version: "); psStringPrepend(&version, "ppVizPSF source: "); Index: trunk/ppViz/src/ppVizPattern/ppVizPatternVersion.c =================================================================== --- trunk/ppViz/src/ppVizPattern/ppVizPatternVersion.c (revision 28041) +++ trunk/ppViz/src/ppVizPattern/ppVizPatternVersion.c (revision 28043) @@ -81,4 +81,6 @@ psString source = ppVizPatternSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "VIZPAT_V", PS_META_REPLACE, NULL, PPVIZPATTERN_VERSION); + psStringPrepend(&version, "ppVizPattern version: "); psStringPrepend(&version, "ppVizPattern source: "); Index: trunk/ppbgrestore/src/ppbgrestoreVersion.c =================================================================== --- trunk/ppbgrestore/src/ppbgrestoreVersion.c (revision 28041) +++ trunk/ppbgrestore/src/ppbgrestoreVersion.c (revision 28043) @@ -64,4 +64,6 @@ psString source = ppbgrestoreSource(); // ppbgrestore software source + psMetadataAddStr(header, PS_LIST_TAIL, "BGREST_V", PS_META_REPLACE, NULL, PPBGRESTORE_VERSION); + psStringPrepend(&version, "ppbgrestore version: "); psStringPrepend(&source, "ppbgrestore source: "); Index: trunk/psLib/src/sys/psConfigure.c =================================================================== --- trunk/psLib/src/sys/psConfigure.c (revision 28041) +++ trunk/psLib/src/sys/psConfigure.c (revision 28043) @@ -66,4 +66,11 @@ } +psString psLibRevision(void) +{ + char *value = NULL; + psStringAppend(&value, "%s", PSLIB_VERSION); + return value; +} + psString psLibSource(void) { Index: trunk/psLib/src/sys/psConfigure.h =================================================================== --- trunk/psLib/src/sys/psConfigure.h (revision 28041) +++ trunk/psLib/src/sys/psConfigure.h (revision 28043) @@ -32,4 +32,12 @@ */ psString psLibVersion(void); + +/** Get current psLib revision number + * + * Returns the current psLib revision number as a string. + * + * @return psString: String with revision number. + */ +psString psLibRevision(void); /** Get current psLib source Index: trunk/psLib/src/types/psMetadataHeader.c =================================================================== --- trunk/psLib/src/types/psMetadataHeader.c (revision 28041) +++ trunk/psLib/src/types/psMetadataHeader.c (revision 28043) @@ -17,5 +17,7 @@ psString version = psLibVersion(); // Software version psString source = psLibSource(); // Software source - + psString revision = psLibRevision(); + psMetadataAddStr(header, PS_LIST_TAIL, "PSLIB_V", PS_META_REPLACE, NULL, revision); + psStringPrepend(&version, "psLib version: "); psStringPrepend(&source, "psLib source: "); @@ -26,5 +28,5 @@ psFree(version); psFree(source); - + psFree(revision); return true; } Index: trunk/psModules/src/config/pmVersion.c =================================================================== --- trunk/psModules/src/config/pmVersion.c (revision 28041) +++ trunk/psModules/src/config/pmVersion.c (revision 28043) @@ -64,4 +64,6 @@ psString source = psModulesSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "MODULE_V", PS_META_REPLACE, NULL, PSMODULES_VERSION); + psStringPrepend(&version, "psModules version: "); psStringPrepend(&source, "psModules source: "); Index: trunk/psModules/src/detrend/Makefile.am =================================================================== --- trunk/psModules/src/detrend/Makefile.am (revision 28041) +++ trunk/psModules/src/detrend/Makefile.am (revision 28043) @@ -8,4 +8,5 @@ pmFringeStats.c \ pmMaskBadPixels.c \ + pmMaskStats.c \ pmNonLinear.c \ pmBias.c \ @@ -27,4 +28,5 @@ pmFringeStats.h \ pmMaskBadPixels.h \ + pmMaskStats.h \ pmNonLinear.h \ pmBias.h \ Index: trunk/psModules/src/detrend/pmMaskStats.c =================================================================== --- trunk/psModules/src/detrend/pmMaskStats.c (revision 28043) +++ trunk/psModules/src/detrend/pmMaskStats.c (revision 28043) @@ -0,0 +1,121 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include +#include +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmFPAview.h" +#include "pmFPAfile.h" +#include "pmFPAfileIO.h" +#include "pmFPAAstrometry.h" + + +# define ESCAPE { \ + psError(PS_ERR_UNKNOWN, false, "I/O failure in pmMaskStats"); \ + psFree (view); \ + return false; \ + } + + + + +bool pmFPAMaskStats(pmFPA *fpa, pmConfig *config) { + PS_ASSERT_PTR_NON_NULL(fpa, false); + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status; + + psU16 staticMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.STATIC"); + psU16 magicMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.MAGIC"); + psU16 dynamicMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.DYNAMIC"); + psU16 advisoryMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.ADVISORY"); + + psS32 Npix_valid = 0; + psS32 Npix_static = 0; + psS32 Npix_magic = 0; + psS32 Npix_dynamic = 0; + psS32 Npix_advisory = 0; + + pmChip *chip; + pmCell *cell; + pmReadout *readout; + pmFPAview *view = pmFPAviewAlloc(0); + while ((chip = pmFPAviewNextChip(view, fpa, 1)) != NULL) { + if (!chip->process || !chip->file_exists) { continue; } + if (!chip->fromFPA) { continue; } + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE; + + while ((cell = pmFPAviewNextCell(view, fpa, 1)) != NULL) { + if (!cell->process || !cell->file_exists) {continue; } + while ((readout = pmFPAviewNextReadout(view, fpa, 1)) != NULL) { + if (!readout->data_exists) {continue; } + + psImage *mask = readout->mask; + if (!pmSingleImageMaskStats(mask,&Npix_valid,&Npix_static,&Npix_magic, + &Npix_dynamic,&Npix_advisory, + staticMaskVal,magicMaskVal, + dynamicMaskVal,advisoryMaskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate masks for readout."); + return(false); + } + psMetadataAddS32(readout->analysis, PS_LIST_TAIL,"MASKFRAC_NPIX", 0, + "Number of valid pixels", Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + } + } + } + return(true); +} + + + +bool pmSingleImageMaskStats(psImage *mask, + psS32 *Npix_valid, psS32 *Npix_static, psS32 *Npix_magic, + psS32 *Npix_dynamic, psS32 *Npix_advisory, + psU16 staticMaskVal, psU16 magicMaskVal, + psU16 dynamicMaskVal, psU16 advisoryMaskVal) { + PS_ASSERT_IMAGE_NON_NULL(mask, false); + *Npix_valid = 0; + *Npix_static = 0; + *Npix_magic = 0; + *Npix_dynamic = 0; + *Npix_advisory = 0; + + psImageMaskType **maskData = mask->data.PS_TYPE_IMAGE_MASK_DATA; + psWarning("HERE: %ld %d %d\n",(long) maskData, mask->numRows,mask->numCols); + for (int i = 0; i < mask->numRows - 1; i++) { + for (int j = 0; j < mask->numCols - 1; j++) { + *Npix_valid += 1; + if (maskData[i][j] & staticMaskVal) { + *Npix_static += 1; + continue; + } + if (maskData[i][j] & dynamicMaskVal) { + *Npix_dynamic += 1; + continue; + } + if (maskData[i][j] & magicMaskVal) { + *Npix_magic += 1; + continue; + } + if (maskData[i][j] & advisoryMaskVal) { + *Npix_advisory += 1; + continue; + } + } + } + return(true); +} + Index: trunk/psModules/src/detrend/pmMaskStats.h =================================================================== --- trunk/psModules/src/detrend/pmMaskStats.h (revision 28043) +++ trunk/psModules/src/detrend/pmMaskStats.h (revision 28043) @@ -0,0 +1,14 @@ +#ifndef PM_MASK_STATS_H +#define PM_MASK_STATS_H + +bool pmFPAMaskStats(pmFPA *fpa, pmConfig *config); +bool pmSingleImageMaskStats(psImage *mask, + psS32 *Npix_valid, psS32 *Npix_static, psS32 *Npix_magic, + psS32 *Npix_dynamic, psS32 *Npix_advisory, + psU16 staticMaskVal, psU16 magicMaskVal, + psU16 dynamicMaskVal, psU16 advisoryMaskVal); + + +//bool pmMaskStats(pmFPA *fpa, pmConfig *config, psMetadata *results); + +#endif Index: trunk/psModules/src/psmodules.h =================================================================== --- trunk/psModules/src/psmodules.h (revision 28041) +++ trunk/psModules/src/psmodules.h (revision 28043) @@ -73,4 +73,5 @@ #include #include +#include #include #include Index: trunk/psastro/src/psastro.h =================================================================== --- trunk/psastro/src/psastro.h (revision 28041) +++ trunk/psastro/src/psastro.h (revision 28043) @@ -71,5 +71,5 @@ bool psastroAstromGuess (int *nStars, pmConfig *config); bool psastroAstromGuessCheck (pmConfig *config); -bool psastroMaskUpdates (pmConfig *config); +bool psastroMaskUpdates (pmConfig *config, psMetadata *stats); bool psastroLoadGlints (pmConfig *config); @@ -179,4 +179,5 @@ bool psastroFPAtoChip (double *xChip, double *yChip, pmFPA *fpa, int nChip, double xFPA, double yFPA); +//bool psastroMaskStats(pmConfig *config, psMetadata *stats); // psastroExtract functions Index: trunk/psastro/src/psastroAnalysis.c =================================================================== --- trunk/psastro/src/psastroAnalysis.c (revision 28041) +++ trunk/psastro/src/psastroAnalysis.c (revision 28043) @@ -154,5 +154,5 @@ } - if (!psastroMaskUpdates (config)) { + if (!psastroMaskUpdates (config, stats)) { psError(psErrorCodeLast(), false, "Failed to generate dynamic masks."); return false; Index: trunk/psastro/src/psastroMaskStats.c =================================================================== --- trunk/psastro/src/psastroMaskStats.c (revision 28043) +++ trunk/psastro/src/psastroMaskStats.c (revision 28043) @@ -0,0 +1,11 @@ + +# include "psastroInternal.h" + +# define ESCAPE { \ + psError(PS_ERR_UNKNOWN, false, "I/O failure in pmMaskStats"); \ + psFree (view); \ + return false; \ + } + + + Index: trunk/psastro/src/psastroMaskUpdates.c =================================================================== --- trunk/psastro/src/psastroMaskUpdates.c (revision 28041) +++ trunk/psastro/src/psastroMaskUpdates.c (revision 28043) @@ -19,8 +19,51 @@ } + +int colStart(psU16 corner_list,int i,int x_0,int y_0,int R,int max) { + if ((corner_list == 0x04)||(corner_list == 0x05)) { + return(0); + } + else { + return((int) fabs(x_0 - sqrt(pow(R,2) - pow(y_0 - i,2)))); + } +} +int colEnd(psU16 corner_list,int i,int x_0,int y_0,int R,int max) { + if ((corner_list == 0x04)||(corner_list == 0x05)) { + return((int) fabs(x_0 - sqrt(pow(R,2) - pow(y_0 - i,2)))); + } + else { + return(max); + } + +} +int rowStart(psU16 corner_list,int j,int x_0,int y_0,int R,int max) { + if ((corner_list == 0x01)||(corner_list == 0x02)|| + (corner_list == 0x03)||(corner_list == 0x0b)|| + (corner_list == 0x07)||(corner_list == 0x0f)) { + return(0); + } + else { + return((int) fabs(y_0 - sqrt(pow(R,2) - pow(x_0 - j,2)))); + } +} +int rowEnd(psU16 corner_list,int j,int x_0,int y_0,int R, int max) { + if (corner_list == 0x0f) { + return(max); + } + else if ((corner_list == 0x01)||(corner_list == 0x02)|| + (corner_list == 0x03)||(corner_list == 0x0b)|| + (corner_list == 0x07)) { + return((int) fabs(y_0 - sqrt(pow(R,2) - pow(x_0 - j,2)))); + } + else { + return(max); + } + +} + /** * create a mask or mask regions based on the collection of reference stars that * are in the vicinity of each chip */ -bool psastroMaskUpdates (pmConfig *config) { +bool psastroMaskUpdates (pmConfig *config, psMetadata *stats) { bool status; @@ -88,4 +131,18 @@ double REFSTAR_MASK_CROSSTALK_MAG_SLOPE= psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_CROSSTALK_MAG_SLOPE"); + // Mask stats variables + psU16 staticMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.STATIC"); + psU16 magicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.MAGIC"); + psU16 dynamicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.DYNAMIC"); + psU16 advisoryMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.ADVISORY"); + + psS32 Npix_valid = 0; + psS32 Npix_static = 0; + psS32 Npix_magic = 0; + psS32 Npix_dynamic = 0; + psS32 Npix_advisory = 0; + + psU16 corner_list = 0x00; + // select the input data sources pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT"); @@ -119,4 +176,12 @@ pmFPA *fpaMask = outMask->fpa; + // Get camera specific mask stat options + psF32 FOV = psMetadataLookupF32(&status, fpaMask->camera, "FOV"); // Almost certainly doesn't exist. Placeholder for now. + psS32 CONSTANT_BLANK = psMetadataLookupS32(&status, fpaMask->camera, "NPIX_INTERCHIP"); // Need to add this as well. + + Npix_valid += CONSTANT_BLANK; + Npix_static += CONSTANT_BLANK; + + // select the reference mask fpa :: we use this to determine cell boundaries pmFPAfile *refMask = psMetadataLookupPtr (NULL, config->files, "PSASTRO.REFMASK"); @@ -363,6 +428,115 @@ } } - } - + } + + // Do the mask stats bit from cell down. + if (psMetadataLookupBool(&status,recipe,"MASK.STATS")) { + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE; + while ((cell = pmFPAviewNextCell(view, fpa, 1)) != NULL) { + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE; + psWarning("In CELL: FOV: %f CONSTANT: %d Mask bits: %d %d %d %d\n", + FOV,CONSTANT_BLANK,staticMaskVal,magicMaskVal,dynamicMaskVal,advisoryMaskVal); + + if (!cell->process || !cell->file_exists) {continue; } + while ((readout = pmFPAviewNextReadout(view, fpa, 1)) != NULL) { + pmReadout *readoutMask = pmFPAviewThisReadout (view, outMask->fpa); + if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_BEFORE)) ESCAPE; + psWarning("In READOUT: FOV: %f CONSTANT: %d Mask bits: %d %d %d %d\n", + FOV,CONSTANT_BLANK,staticMaskVal,magicMaskVal,dynamicMaskVal,advisoryMaskVal); + + if (!readoutMask->data_exists) {continue; } + psPlane coordFPA; + psPlane coordCell; + + psImage *mask = readoutMask->mask; + if (!mask) {continue;} + psImageMaskType **maskData = mask->data.PS_TYPE_IMAGE_MASK_DATA; + // Dance coordinates around + // Calculate which corners fall within the field of view. + // 0x04 0x08 + // 0x01 0x02 + + coordFPA.x = 0.0; + coordFPA.y = 0.0; + psPlaneTransformApply(&coordCell,chip->fromFPA,&coordFPA); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV,2)) { + corner_list = corner_list | 0x01; + } + + coordCell.x = mask->numCols - 1; + psPlaneTransformApply(&coordCell,chip->fromFPA,&coordFPA); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV,2)) { + corner_list = corner_list | 0x02; + } + + coordCell.y = mask->numRows - 1; + psPlaneTransformApply(&coordCell,chip->fromFPA,&coordFPA); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV,2)) { + corner_list = corner_list | 0x08; + } + + coordCell.x = 0; + psPlaneTransformApply(&coordCell,chip->fromFPA,&coordFPA); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV,2)) { + corner_list = corner_list | 0x04; + } + + int x_0 = coordFPA.x; + int y_0 = coordFPA.y; + + + // Scan over the valid regions of the image and count masked pixels + if ((corner_list == 0x04)||(corner_list == 0x08)|| + (corner_list == 0x05)||(corner_list == 0x0a)) { + for (int i = 0; i < mask->numRows - 1; i++) { + for (int j = colStart(corner_list,i,x_0,y_0,FOV,mask->numCols - 1); + j < colEnd(corner_list,i,x_0,y_0,FOV,mask->numCols - 1); j++) { + Npix_valid++; + if (maskData[i][j] & staticMaskVal) { + Npix_static++; + continue; + } + if (maskData[i][j] & dynamicMaskVal) { + Npix_dynamic++; + continue; + } + if (maskData[i][j] & magicMaskVal) { + Npix_magic++; + continue; + } + if (maskData[i][j] & advisoryMaskVal) { + Npix_advisory++; + continue; + } + } + } + } + else { + for (int j = 0; j < mask->numCols - 1; j++) { + for (int i = rowStart(corner_list,j,x_0,y_0,FOV,mask->numRows - 1); + i < rowEnd(corner_list,j,x_0,y_0,FOV,mask->numRows - 1); i++) { + Npix_valid++; + if (maskData[i][j] & staticMaskVal) { + Npix_static++; + continue; + } + if (maskData[i][j] & dynamicMaskVal) { + Npix_dynamic++; + continue; + } + if (maskData[i][j] & magicMaskVal) { + Npix_magic++; + continue; + } + if (maskData[i][j] & advisoryMaskVal) { + Npix_advisory++; + continue; + } + } + } + } + } + } + } // output sequence for mask corresponding to this chip (XXX this may not be needed...) *viewMask = *view; @@ -392,4 +566,15 @@ } + psMetadataAddS32(stats,PS_LIST_TAIL, "MASKFRAC_NPIX", 0, + "Number of valid pixels", Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + // deactivate all files pmFPAfileActivate (config->files, false, NULL); Index: trunk/psastro/src/psastroVersion.c =================================================================== --- trunk/psastro/src/psastroVersion.c (revision 28041) +++ trunk/psastro/src/psastroVersion.c (revision 28043) @@ -58,4 +58,6 @@ psString source = psastroSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "ASTRO_V", PS_META_REPLACE, NULL, PSASTRO_VERSION); + psStringPrepend(&version, "psastro version: "); psStringPrepend(&source, "psastro source: "); Index: trunk/psphot/src/psphotVersion.c =================================================================== --- trunk/psphot/src/psphotVersion.c (revision 28041) +++ trunk/psphot/src/psphotVersion.c (revision 28043) @@ -69,4 +69,6 @@ psString source = psphotSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "PHOT_V", PS_META_REPLACE, NULL, PSPHOT_VERSION); + psStringPrepend(&version, "psphot version: "); psStringPrepend(&source, "psphot source: "); Index: trunk/pstamp/scripts/detect_query_create =================================================================== --- trunk/pstamp/scripts/detect_query_create (revision 28041) +++ trunk/pstamp/scripts/detect_query_create (revision 28043) @@ -61,5 +61,10 @@ comment => 'site identifier (MPC observatory code)', value => undef - } + }, + { name => 'STAGE', + writetype => TSTRING, + comment => 'processing stage to examine', + value => undef + } ]; unless(defined($nostage)) { Index: trunk/pstamp/test/detect_query_create =================================================================== --- trunk/pstamp/test/detect_query_create (revision 28043) +++ trunk/pstamp/test/detect_query_create (revision 28043) @@ -0,0 +1,287 @@ +#!/usr/bin/env perl + +# create a MOPS_DETECTABILITY_QUERY fits table from a text description file + +use warnings; +use strict; + +use Astro::FITS::CFITSIO qw( :constants ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for output table + +my ( $input, # Name of input text file + $output, # Name of output table + $query_id, + $nostage, + ); + +GetOptions( + 'input|i=s' => \$input, + 'output|o=s' => \$output, + 'query_id|q=s' => \$query_id, + 'nostage' => \$nostage, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --input --output", + -exitval => 3) unless defined $input and defined $output; + +# The header kewords +my $header = [ + { name => 'QUERY_ID', + writetype => TSTRING, + comment => 'MOPS Query ID for this batch query', + value => undef + }, + { name => 'EXTVER', + writetype => TSTRING, + comment => 'Extension version', + value => undef + }, + { name => 'FPA_ID', + writetype => TSTRING, + comment => 'orginal FPA_ID used at ingest', + value => undef + }, + { name => 'MJD-OBS', + writetype => TDOUBLE, + comment => 'starting time of the exposure, MJD', + value => undef + }, + { name => 'FILTER', + writetype => TSTRING, + comment => 'effective filter use for the exposure', + value => undef + }, + { name => 'OBSCODE', + writetype => TSTRING, + comment => 'site identifier (MPC observatory code)', + value => undef + }, +# { name => 'STAGE', +# writetype => TSTRING, +# comment => 'processing stage to examine', +# value => undef +# } +]; +unless(defined($nostage)) { + push @{ $header }, { name => 'STAGE', + writetype => TSTRING, + comment => 'processing stage to examine', + value => undef + }; +} + +# Specification of columns to write +my $columns = [ + # matching rownum from detectability original request + { name => 'ROWNUM', type => '20A', writetype => TSTRING }, + # coordinate at start of exposure, in degrees + { name => 'RA1_DEG', type => 'D', writetype => TDOUBLE }, + # coordinate at start of exposure, in degrees + { name => 'DEC1_DEG', type => 'D', writetype => TDOUBLE }, + # coordinate at end of exposure, in degrees + { name => 'RA2_DEG', type => 'D', writetype => TDOUBLE }, + # coordinate at end of exposure, in degrees + { name => 'DEC2_DEG', type => 'D', writetype => TDOUBLE }, + # apparent magnitude + { name => 'MAG', type => 'D', writetype => TDOUBLE }, +]; + +my $in; +if ($input eq '-') { + $in = \*STDIN; +} else { + open $in, "<$input" or die "cannot open $input for reading"; +} + +my @colData; +foreach (@$columns) { + push @colData, []; +} + + +my $numRows = read_data_for_table($in,'\s+', \@colData, $header); +if (!$numRows) { + print STDERR "no data in $input\n"; + exit 1; +} + +# overwrite the QUERY_ID value from the input file with the command +# line argument + +if ($query_id) { + $header->[0]->{value} = $query_id; +} + +my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header); + +exit $status; + +# XXXXX: This should be in a module +# two utility functions that may be used to create a FITS binary +# table from hashes describing the header keywords and columns + +# read_table_description reads the data for a table from a simple text file +# make_fits_table writes out the table to a named file + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +# A function to build a fits binary table from supplied data +# +sub make_fits_table { + my $output = shift; # name of output file + my $extname = shift; # extension name + my $numRows = shift; # number of rows in the table + my $colData = shift; # ref to array of arrays containing the data for each column + my $columns = shift; # ref to array of column descriptions (each a hash) + # with keys: name, type, and writetype + my $header = shift; # ref to array of header keyword descriptions - each a hash + # with keys: name, name, writetype, comment, and value + my $status = 0; + + die "incorrect arguments" if !defined($columns); + # note $header can be nil + + # build arrays for cfitsio + my @colNames; # Names of columns + my @colTypes; # Types of columns + my @colWriteType; # type to use to write + + foreach my $colSpec ( @$columns) { + push @colNames, $colSpec->{name}; + push @colTypes, $colSpec->{type}; + push @colWriteType, $colSpec->{writetype}; + } + + if (-e $output) { + unlink "$output" or die "failed to remove existing $output"; + } + + my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); # Output file handle + check_fitsio( $status ); + + $outFits->create_img( 16, 0, undef, $status ); + check_fitsio( $status ); + + # Create the table + + $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, + \@colNames, \@colTypes, undef, $extname, $status ); + check_fitsio( $status ); + + # if header keyword descriptions were provided add them + if ($header) { + foreach my $headerword ( @$header ) { + my $value = $headerword->{value}; + unless (defined $value) { + print "Can't find header keyword $headerword\n"; + next; + } + # zap quotation marks + $value =~ s/\'//g; + my $name = $headerword->{name}; + my $type = $headerword->{writetype}; + my $comment = $headerword->{comment}; + $outFits->write_key( $type, $name, $value, $comment, $status ); + check_fitsio( $status ); + } + } + + + for (my $i = 0; $i < scalar @colNames; $i++) { + my $writeType = $colWriteType[$i]; + $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData->[$i], $status ); + check_fitsio( $status ); + } + + $outFits->close_file( $status ); + + return 0; + +} # end of sub make_fits_table + + + +# read the table contents from a file +# +# input text file format: +# lines that begin with '#' are comment lines and are skipped. +# other lines are data. Each data line is split into fields with the +# provided separator +# +# if $header is not null header the first non-commented line is read to +# fill the value for each header keyword. The number of fields must match +# the number of keywords. +# +# Following the optional header data, each data line contains data for each +# row in the table. The number of fields must match the number of column +# arrays provided. + +sub read_data_for_table { + my $in = shift; # input file handle + my $sep = shift; # string containing field separator + my $colData = shift; # reference to an array of arrays for the data + my $header = shift; # rerence to array of header keyword descriptions + + my $line_num = 0; + + # read data for header if any data is expected + if ($header) { + my $nhead = @$header; + while (my $line = <$in>) { + $line_num++; + chomp $line; + next if !$line; # skip blank lines + next if ($line =~ /^#/); # skip comment lines + my @vals = split /$sep/, $line; + my $nvals = @vals; + die "number of header columns in input $nvals does not equal expected number of header words $nhead" + if (@vals != @$header); + + for (my $i=0; $i < @$header; $i++) { + $header->[$i]->{value} = $vals[$i]; + } + + last; # only one header line + } + } + + my $row_num = 0; + my $ncols = @$colData; + while (my $line = <$in>) { + chomp $line; + $line_num++; + next if !$line; # skip blank lines + next if ($line =~ /^#/); # skip comment lines + + my @vals = split /$sep/, $line; + my $nvals = @vals; + die "number of columns $nvals in input does not equal expected number of header " + . " words $ncols on line $line_num" if ($nvals != $ncols); + + for (my $col = 0; $col < @$colData; $col++) { + $colData->[$col]->[$row_num] = $vals[$col]; + } + $row_num++; + } + + # we return the number of rows read + return $row_num; +} + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + die "CFITSIO error: $msg\n"; + } +} Index: trunk/pswarp/src/Makefile.am =================================================================== --- trunk/pswarp/src/Makefile.am (revision 28041) +++ trunk/pswarp/src/Makefile.am (revision 28043) @@ -34,4 +34,5 @@ pswarpErrorCodes.c \ pswarpMapGrid.c \ + pswarpMaskStats.c \ pswarpMatchRange.c \ pswarpParseCamera.c \ Index: trunk/pswarp/src/pswarp.h =================================================================== --- trunk/pswarp/src/pswarp.h (revision 28041) +++ trunk/pswarp/src/pswarp.h (revision 28043) @@ -109,4 +109,9 @@ ); +bool pswarpMaskStats(const pmReadout *readout, + psMetadata *stats, + const pmConfig *config); + + /** * define threads for this program Index: trunk/pswarp/src/pswarpLoop.c =================================================================== --- trunk/pswarp/src/pswarpLoop.c (revision 28041) +++ trunk/pswarp/src/pswarpLoop.c (revision 28043) @@ -266,5 +266,13 @@ goto DONE; } - + bool doStats = psMetadataLookupBool(&mdok,recipe,"MASK.STATS"); + if (doStats) { + if (!pswarpMaskStats(output, stats, config)) { + psError(psErrorCodeLast(), false, "Unable to calculate mask stats."); + psFree(cells); + psFree(view); + goto DONE; + } + } // Set covariance matrix for output { @@ -332,5 +340,5 @@ pswarpVersionHeader(hdu->header); - + if (!pmAstromWriteWCS(hdu->header, outFPA, outChip, WCS_NONLIN_TOL)) { psError(psErrorCodeLast(), false, "Unable to generate WCS header."); @@ -347,4 +355,5 @@ pswarpFileActivation(config, detectorFiles, false); pswarpFileActivation(config, independentFiles, false); + // We need a new PSF model for the warped frame. It would be good to generate this analytically, but @@ -421,4 +430,5 @@ } } + // Add MD5 information for readout Index: trunk/pswarp/src/pswarpMaskStats.c =================================================================== --- trunk/pswarp/src/pswarpMaskStats.c (revision 28043) +++ trunk/pswarp/src/pswarpMaskStats.c (revision 28043) @@ -0,0 +1,45 @@ +#include "pswarp.h" +#include + +bool pswarpMaskStats(const pmReadout *readout, psMetadata *stats, const pmConfig *config) { + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status; + psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, PSWARP_RECIPE); + psU16 staticMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.STATIC"); + psU16 magicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.MAGIC"); + psU16 dynamicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.DYNAMIC"); + psU16 advisoryMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.ADVISORY"); + + psS32 Npix_valid = 0; + psS32 Npix_static = 0; + psS32 Npix_magic = 0; + psS32 Npix_dynamic = 0; + psS32 Npix_advisory = 0; + + if (!readout || !readout->data_exists) { + // Nothing to process + return(true); + } + psWarning("pswarpMaskStats: %d %d %d %d\n",staticMaskVal,magicMaskVal,dynamicMaskVal,advisoryMaskVal); + psImage *mask = readout->mask; // Mask of interest; + if (!pmSingleImageMaskStats(mask,&Npix_valid,&Npix_static,&Npix_magic, + &Npix_dynamic,&Npix_advisory, + staticMaskVal,magicMaskVal, + dynamicMaskVal,advisoryMaskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate masks for readout."); + return(false); + } + psMetadataAddS32(stats, PS_LIST_TAIL,"MASKFRAC_NPIX", 0, + "Number of valid pixels", Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + return(true); +} Index: trunk/pswarp/src/pswarpVersion.c =================================================================== --- trunk/pswarp/src/pswarpVersion.c (revision 28041) +++ trunk/pswarp/src/pswarpVersion.c (revision 28043) @@ -85,4 +85,6 @@ psString source = pswarpSource(); // Software source + psMetadataAddStr(header, PS_LIST_TAIL, "WARP_V", PS_META_REPLACE, NULL, PSWARP_VERSION); + psStringPrepend(&version, "pswarp version: "); psStringPrepend(&source, "pswarp source: ");