Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 28041) +++ trunk/ippTools/share/Makefile.am (revision 28043) @@ -28,4 +28,5 @@ chiptool_change_imfile_data_state.sql \ chiptool_completely_processed_exp.sql \ + chiptool_coalesce_run.sql \ chiptool_definecopy.sql \ chiptool_donecleanup.sql \ @@ -97,4 +98,5 @@ difftool_change_run_state.sql \ difftool_completed_runs.sql \ + difftool_coalesce_run.sql \ difftool_definewarpstack_part1.sql \ difftool_definewarpstack_part2.sql \ @@ -298,4 +300,5 @@ warptool_finished_run_select.sql \ warptool_finish_run.sql \ + warptool_coalesce_run.sql \ warptool_imfile.sql \ warptool_listrun.sql \ Index: trunk/ippTools/share/camtool_queue_chip_id.sql =================================================================== --- trunk/ippTools/share/camtool_queue_chip_id.sql (revision 28041) +++ trunk/ippTools/share/camtool_queue_chip_id.sql (revision 28043) @@ -17,4 +17,10 @@ '%s', -- end_stage %lld, -- magicked + '%s', -- software ver + '%s', -- maskfrac_npix + '%s', -- _static + '%s', -- _dynamic + '%s', -- _magic + '%s', -- _advisory '%s' -- note FROM chipRun Index: trunk/ippTools/share/chiptool_coalesce_run.sql =================================================================== --- trunk/ippTools/share/chiptool_coalesce_run.sql (revision 28043) +++ trunk/ippTools/share/chiptool_coalesce_run.sql (revision 28043) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from chipProcessedImfile where data_state = 'full' AND fault = 0 AND chip_id = %lld Index: trunk/ippTools/share/difftool_coalesce_run.sql =================================================================== --- trunk/ippTools/share/difftool_coalesce_run.sql (revision 28043) +++ trunk/ippTools/share/difftool_coalesce_run.sql (revision 28043) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from diffSkyfile where diff_id = %lld Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 28041) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 28043) @@ -276,4 +276,10 @@ end_stage VARCHAR(64), magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, note VARCHAR(255), PRIMARY KEY(chip_id), @@ -359,4 +365,10 @@ fault SMALLINT NOT NULL, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, PRIMARY KEY(chip_id, exp_id, class_id), KEY(data_state), @@ -387,4 +399,10 @@ end_stage VARCHAR(64), magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, note VARCHAR(255), PRIMARY KEY(cam_id), @@ -464,4 +482,10 @@ quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT NOT NULL, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, PRIMARY KEY(cam_id), KEY(fault), @@ -867,4 +891,10 @@ registered DATETIME, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, note VARCHAR(255), PRIMARY KEY(warp_id), @@ -923,4 +953,10 @@ fault SMALLINT, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, PRIMARY KEY(warp_id, skycell_id, tess_id), KEY(good_frac), @@ -948,4 +984,5 @@ tess_id VARCHAR(64), filter VARCHAR(64), + software_ver VARCHAR(16), note VARCHAR(255), PRIMARY KEY(stack_id), @@ -995,4 +1032,5 @@ quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT, + software_ver VARCHAR(16), PRIMARY KEY(stack_id), KEY(dtime_stack), @@ -1017,4 +1055,10 @@ exposure TINYINT DEFAULT 0, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, diff_mode SMALLINT NOT NULL, note VARCHAR(255), @@ -1080,4 +1124,10 @@ fault SMALLINT, magicked BIGINT, + software_ver VARCHAR(16), + maskfrac_npix INT, + maskfrac_static FLOAT, + maskfrac_dynamic FLOAT, + maskfrac_magic FLOAT, + maskfrac_advisory FLOAT, PRIMARY KEY(diff_id, skycell_id), KEY(good_frac), Index: trunk/ippTools/share/warptool_coalesce_run.sql =================================================================== --- trunk/ippTools/share/warptool_coalesce_run.sql (revision 28043) +++ trunk/ippTools/share/warptool_coalesce_run.sql (revision 28043) @@ -0,0 +1,1 @@ +select software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory from warpSkyfile where warp_id = %lld Index: trunk/ippTools/src/camtool.c =================================================================== --- trunk/ippTools/src/camtool.c (revision 28041) +++ trunk/ippTools/src/camtool.c (revision 28043) @@ -481,4 +481,40 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_ppimage, config->args, "-ver_ppimage", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_psastro) { + software_ver = pxMergeCodeVersions(software_ver,ver_psastro); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + if (ver_ppimage) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppimage); + } + if (ver_streaks) { + software_ver = pxMergeCodeVersions(software_ver,ver_streaks); + } // Get this from the chipRun @@ -595,5 +631,11 @@ path_base, fault, - quality + software_ver, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory, + quality ); @@ -620,4 +662,29 @@ // else continue on... + + // Set camRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "camRun", "cam_id", cam_id, software_ver)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set camRun.software_ver for cam_id: %" PRId64, + cam_id); + psFree(output); + return(false); + } + // Set chipRun.maskfrac* to the appropriate values. + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "camRun", "cam_id",cam_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set camRun.maskstats for cam_id: %" PRId64, + cam_id); + psFree(output); + return(false); + } + } + // since there is only one exp per 'new' set camRun.state = 'full' // propagate magicked state from chipRun @@ -628,4 +695,5 @@ return false; } + psFree(row); Index: trunk/ippTools/src/camtoolConfig.c =================================================================== --- trunk/ippTools/src/camtoolConfig.c (revision 28041) +++ trunk/ippTools/src/camtoolConfig.c (revision 28043) @@ -174,4 +174,19 @@ psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_ppimage", 0, "define ppImage version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -processedexp psMetadata *processedexpArgs = psMetadataAlloc(); Index: trunk/ippTools/src/chiptool.c =================================================================== --- trunk/ippTools/src/chiptool.c (revision 28041) +++ trunk/ippTools/src/chiptool.c (revision 28043) @@ -585,4 +585,40 @@ PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_ppimage, config->args, "-ver_ppimage", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_psastro) { + ver_code = pxMergeCodeVersions(ver_code,ver_psastro); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_ppimage) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppimage); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } + // default values PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -661,5 +697,11 @@ fault, quality, - magicked + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { // rollback @@ -1353,4 +1395,32 @@ psMetadata *row = output->data[i]; + psS64 chip_id = psMetadataLookupS64(NULL,row,"chip_id"); + + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"chiptool_coalesce_run.sql",chip_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + psMetadataAddStr(row, PS_LIST_TAIL, "software_ver", 0, "Software version", software_ver); + psMetadataAddS32(row, PS_LIST_TAIL, "maskfrac_npix", 0, "Number of pixels used for mask stats", maskfrac_npix); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_static", 0, "Fraction of static masked pixels", maskfrac_static); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_dynamic", 0, "Fraction of dynamic masked pixels", maskfrac_dynamic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_magic", 0, "Fraction of magic masked pixels", maskfrac_magic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_advisory", 0, "Fraction of advisory pixels", maskfrac_advisory); + psWarning("ADVANCE %ld %s %d %f %f %f %f\n",chip_id,software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory); chipRunRow *chipRun = chipRunObjectFromMetadata(row); if (!psDBTransaction(config->dbh)) { @@ -1359,4 +1429,25 @@ } + // Set chipRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "chipRun", "chip_id", chip_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set chipRun.maskfrac* to the appropriate values. + if (!pxSetRunMaskfrac(config, "chipRun", "chip_id",chip_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } // set chipRun.state to 'stop' and update the magicked state if (!pxchipRunSetState(config, chipRun->chip_id, "full", chipRun->magicked)) { @@ -1383,16 +1474,16 @@ // stop if (!pxcamQueueByChipID(config, - chipRun->chip_id, - chipRun->workdir, - chipRun->label, - chipRun->data_group, - chipRun->dist_group, - chipRun->reduction, - chipRun->expgroup, - chipRun->dvodb, - chipRun->tess_id, - chipRun->end_stage, - chipRun->magicked, - NULL // note does not propagate + chipRun->chip_id, + chipRun->workdir, + chipRun->label, + chipRun->data_group, + chipRun->dist_group, + chipRun->reduction, + chipRun->expgroup, + chipRun->dvodb, + chipRun->tess_id, + chipRun->end_stage, + chipRun->magicked, + NULL // note does not propagate )) { if (!psDBRollback(config->dbh)) { Index: trunk/ippTools/src/chiptoolConfig.c =================================================================== --- trunk/ippTools/src/chiptoolConfig.c (revision 28041) +++ trunk/ippTools/src/chiptoolConfig.c (revision 28043) @@ -181,4 +181,18 @@ psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", 0); + + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_ppimage", 0, "define ppImage version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); // -processedimfile Index: trunk/ippTools/src/difftool.c =================================================================== --- trunk/ippTools/src/difftool.c (revision 28041) +++ trunk/ippTools/src/difftool.c (revision 28043) @@ -162,4 +162,10 @@ exposure, false, + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory diff_mode, note @@ -549,4 +555,36 @@ PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_ppsub, config->args, "-ver_ppsub", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s ppsub %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_ppsub,ver_streaks); + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_ppsub) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppsub); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } if (!psDBTransaction(config->dbh)) { @@ -582,5 +620,11 @@ fault, quality, - magicked + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { if (!psDBRollback(config->dbh)) { @@ -671,4 +715,48 @@ psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"difftool_coalesce_run.sql",diff_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set diffRun.software_ver to the appropriate value + if (software_ver) { + if (!pxSetRunSoftware(config, "diffRun", "diff_id", diff_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set diffRun.software_ver for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + // Set diffRun.maskfrac* to the appropriate values. + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "diffRun", "diff_id", diff_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set diffRun.software_ver for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } // set diffRun.state to 'full' if (!setdiffRunState(config, diff_id, "full", magicked)) { @@ -985,4 +1073,10 @@ false, 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory diff_mode, // diff_mode note @@ -1412,4 +1506,10 @@ true, // exposure 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_WARP_STACK, note @@ -1764,4 +1864,10 @@ true, // exposure false, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_WARP_WARP, note); // Run to insert @@ -2046,4 +2152,10 @@ false, // exposure 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_STACK_STACK, // diff_mode note Index: trunk/ippTools/src/difftoolConfig.c =================================================================== --- trunk/ippTools/src/difftoolConfig.c (revision 28041) +++ trunk/ippTools/src/difftoolConfig.c (revision 28043) @@ -126,4 +126,15 @@ psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_ppsub", 0, "define ppSub version", NULL); + psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); // -advance Index: trunk/ippTools/src/pxcam.c =================================================================== --- trunk/ippTools/src/pxcam.c (revision 28041) +++ trunk/ippTools/src/pxcam.c (revision 28043) @@ -152,5 +152,5 @@ char *tess_id, char *end_stage, - psS64 magicked, + psS64 magicked, char *note) { @@ -184,4 +184,10 @@ end_stage ? end_stage : "NULL", (long long) magicked, + "NULL", // This shouldn't be propagated + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", note ? note : "NULL", (long long)chip_id Index: trunk/ippTools/src/pxchip.c =================================================================== --- trunk/ippTools/src/pxchip.c (revision 28041) +++ trunk/ippTools/src/pxchip.c (revision 28043) @@ -277,4 +277,10 @@ end_stage, 0, // magicked + NULL, + 0, + NAN, + NAN, + NAN, + NAN, note ) Index: trunk/ippTools/src/pxtools.c =================================================================== --- trunk/ippTools/src/pxtools.c (revision 28041) +++ trunk/ippTools/src/pxtools.c (revision 28043) @@ -46,4 +46,119 @@ if (!strcmp(state, "scrubbed")) return true; return false; +} + +psString pxMergeCodeVersions(psString version1, psString version2) { + psString out = NULL; + + bool mod1 = false; + bool mod2 = false; + + psS32 num1; + psS32 num2; + psS32 numO; + + if (!version1) { + psStringAppend(&out,version2); + return(out); + } + if (!version2) { + psStringAppend(&out,version1); + return(out); + } + + if (strchr(version1,'M')) { + psStringSubstitute(&version1,"M",""); + mod1 = true; + } + if (strchr(version2,'M')) { + psStringSubstitute(&version2,"M",""); + mod2 = true; + } + + num1 = strtol(version1,NULL,10); + num2 = strtol(version2,NULL,10); + + if (num1 >= num2) { + numO = num1; + } + else { + numO = num2; + } + + psStringAppend(&out,"%" PRId32,numO); + if (mod1 || mod2) { + psStringAppend(&out,"M"); + } + return(out); +} + +bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, + psS32 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psF32 *maskfrac_magic, psF32 *maskfrac_advisory) { + psString query = pxDataGet(dbQFile); +/* psString text_id = NULL; */ +/* psStringAppend(&text_id," %" PRId64,stage_id); */ +/* psStringSubstitute(&query,text_id,"@STAGE_ID@"); */ +/* psFree(text_id); */ + if (!p_psDBRunQueryF(config->dbh, query, stage_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS32 this_npix = psMetadataLookupS32(NULL, row, "maskfrac_npix"); + psF32 this_static = psMetadataLookupF32(NULL, row, "maskfrac_static"); + psF32 this_dynamic = psMetadataLookupF32(NULL, row, "maskfrac_dynamic"); + psF32 this_magic = psMetadataLookupF32(NULL, row, "maskfrac_magic"); + psF32 this_advisory = psMetadataLookupF32(NULL, row, "maskfrac_advisory"); + psString this_version = psMetadataLookupStr(NULL, row, "software_ver"); + + *software_ver = pxMergeCodeVersions(*software_ver,this_version); + + *maskfrac_static = ((*maskfrac_static * *maskfrac_npix) + (this_npix * this_static)) / (this_npix + *maskfrac_npix); + *maskfrac_dynamic = ((*maskfrac_dynamic * *maskfrac_npix) + (this_npix * this_dynamic)) / (this_npix + *maskfrac_npix); + *maskfrac_magic = ((*maskfrac_magic * *maskfrac_npix) + (this_npix * this_magic)) / (this_npix + *maskfrac_npix); + *maskfrac_advisory = ((*maskfrac_advisory * *maskfrac_npix) + (this_npix * this_advisory)) / (this_npix + *maskfrac_npix); + *maskfrac_npix += this_npix; + } + psFree(output); + return(true); +} + +bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psString software_ver) { + char *query = "UPDATE %s SET software_ver = '%s' WHERE %s = %" PRId64; + printf(query,tableName,software_ver,stage_id_name,stage_id); + if (!p_psDBRunQueryF(config->dbh,query,tableName,software_ver,stage_id_name,stage_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set software version for %s %" PRId64,stage_id_name,stage_id); + return(false); + } + + return(true); +} +bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psS32 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psF32 maskfrac_magic, psF32 maskfrac_advisory) { + char *query = "UPDATE %s SET maskfrac_npix = %d, maskfrac_static = %f, maskfrac_dynamic = %f, maskfrac_magic = %f, maskfrac_advisory = %f WHERE %s = %" PRId64; + if (!p_psDBRunQueryF(config->dbh,query,tableName,maskfrac_npix,maskfrac_static, + maskfrac_dynamic, maskfrac_magic,maskfrac_advisory,stage_id_name,stage_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set maskfrac stats for %s %" PRId64,stage_id_name,stage_id); + return(false); + } + + + + return(true); } Index: trunk/ippTools/src/pxtools.h =================================================================== --- trunk/ippTools/src/pxtools.h (revision 28041) +++ trunk/ippTools/src/pxtools.h (revision 28043) @@ -53,4 +53,14 @@ bool pxIsValidState(const char *state); +psString pxMergeCodeVersions(psString version1, psString version2); +bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, + psS32 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psF32 *maskfrac_magic, psF32 *maskfrac_advisory); +bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psString software_ver); +bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psS32 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psF32 maskfrac_magic, psF32 maskfrac_advisory); + bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows); bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *field, char *name, char *op); Index: trunk/ippTools/src/pxwarp.c =================================================================== --- trunk/ippTools/src/pxwarp.c (revision 28041) +++ trunk/ippTools/src/pxwarp.c (revision 28043) @@ -60,4 +60,10 @@ NULL, // registered 0, // magicked set to zero when created may get updated when warpRun goes to 'full' + NULL, // version + 0, // maskfrac_npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory NULL // note )) { Index: trunk/ippTools/src/stacktool.c =================================================================== --- trunk/ippTools/src/stacktool.c (revision 28041) +++ trunk/ippTools/src/stacktool.c (revision 28043) @@ -370,4 +370,5 @@ tess_id, filter, + NULL, // software_ver note); @@ -526,4 +527,5 @@ tess_id, filter, + NULL, // software_ver note); @@ -844,4 +846,30 @@ PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_ppstack, config->args, "-ver_ppstack", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s ppstack %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_ppstack,ver_streaks); + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + if (ver_ppstack) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstack); + } + if (ver_streaks) { + software_ver = pxMergeCodeVersions(software_ver,ver_streaks); + } + // default values PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -883,4 +911,5 @@ good_frac, fault, + software_ver, quality )) { @@ -893,4 +922,14 @@ if (fault == 0) { + // Set stackRun software if we are finished. + if (!pxSetRunSoftware(config, "stackRun", "stack_id", stack_id, software_ver)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set stackRun.software_ver for stack_id: %" PRId64, + stack_id); + return(false); + } + if (!setstackRunState(config, stack_id, "full")) { if (!psDBRollback(config->dbh)) { Index: trunk/ippTools/src/stacktoolConfig.c =================================================================== --- trunk/ippTools/src/stacktoolConfig.c (revision 28041) +++ trunk/ippTools/src/stacktoolConfig.c (revision 28043) @@ -171,4 +171,11 @@ psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_ppstack", 0, "define ppStack version", NULL); + psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + // -sumskyfile psMetadata *sumskyfileArgs= psMetadataAlloc(); Index: trunk/ippTools/src/warptool.c =================================================================== --- trunk/ippTools/src/warptool.c (revision 28041) +++ trunk/ippTools/src/warptool.c (revision 28043) @@ -177,5 +177,11 @@ registered, 0, // magicked - note + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory + note ); if (!warpRun) { @@ -1019,8 +1025,42 @@ PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_STR(ver_pswarp, config->args, "-ver_pswarp", false, false); + PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s pswarp %s streaks %s\n", + ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_pswarp,ver_streaks); + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_pswarp) { + ver_code = pxMergeCodeVersions(ver_code,ver_pswarp); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } + // default values PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + // we don't want to insert the last skyfile in a run but then not mark the // run as 'stop' @@ -1050,5 +1090,11 @@ fault, quality, - magicked + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { if (!psDBRollback(config->dbh)) { @@ -1115,4 +1161,47 @@ bool status; psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); + + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"warptool_coalesce_run.sql",warp_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set warpRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "warpRun", "warp_id", warp_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, + warp_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set warpRun.maskfrac* to the appropriate values. + if (!pxSetRunMaskfrac(config, "warpRun", "warp_id", warp_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, + warp_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + + if (!status) { psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id"); Index: trunk/ippTools/src/warptoolConfig.c =================================================================== --- trunk/ippTools/src/warptoolConfig.c (revision 28041) +++ trunk/ippTools/src/warptoolConfig.c (revision 28043) @@ -224,4 +224,18 @@ psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_pswarp", 0, "define pswarp version", NULL); + psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -warped psMetadata *warpedArgs = psMetadataAlloc();