Index: branches/czw_branch/20100427/ippTools/src/camtool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/camtool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/camtool.c (revision 28017) @@ -490,7 +490,32 @@ PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); - + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_psastro) { + software_ver = pxMergeCodeVersions(software_ver,ver_psastro); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + if (ver_ppimage) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppimage); + } + if (ver_streaks) { + software_ver = pxMergeCodeVersions(software_ver,ver_streaks); + } + // Get this from the chipRun // PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); @@ -606,13 +631,11 @@ path_base, fault, - quality -/* quality, */ -/* ver_pslib, */ -/* ver_psmodules, */ -/* ver_psphot, */ -/* ver_psastro, */ -/* ver_ppstats, */ -/* ver_ppimage, */ -/* ver_streaks */ + software_ver, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory, + quality ); @@ -639,4 +662,29 @@ // else continue on... + + // Set camRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "camRun", "cam_id", cam_id, software_ver)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set camRun.software_ver for cam_id: %" PRId64, + cam_id); + psFree(output); + return(false); + } + // Set chipRun.maskfrac* to the appropriate values. + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "camRun", "cam_id",cam_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set camRun.maskstats for cam_id: %" PRId64, + cam_id); + psFree(output); + return(false); + } + } + // since there is only one exp per 'new' set camRun.state = 'full' // propagate magicked state from chipRun @@ -647,4 +695,5 @@ return false; } + psFree(row); Index: branches/czw_branch/20100427/ippTools/src/camtoolConfig.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/camtoolConfig.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/camtoolConfig.c (revision 28017) @@ -179,7 +179,14 @@ psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); - psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_ppImage", 0, "define ppImage version", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_ppimage", 0, "define ppImage version", NULL); psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); - + + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -processedexp psMetadata *processedexpArgs = psMetadataAlloc(); Index: branches/czw_branch/20100427/ippTools/src/chiptool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/chiptool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/chiptool.c (revision 28017) @@ -422,8 +422,8 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); // we only allow a single label to match (do not use pxAddLabelSearchArgs here) - PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE"); PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); - PXOPT_COPY_STR(config->args, where, "-dist_group", "chipRun.dist_group", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "chipRun.dist_group", "LIKE"); if (!psListLength(where->list)) { @@ -471,4 +471,6 @@ } psFree(where); + + psStringAppend(&query, "\nORDER BY priority DESC, chip_id"); // treat limit == 0 as "no limit" @@ -591,6 +593,31 @@ PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_psastro) { + ver_code = pxMergeCodeVersions(ver_code,ver_psastro); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_ppimage) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppimage); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } // default values @@ -670,13 +697,11 @@ fault, quality, - magicked -/* magicked, */ -/* ver_pslib, */ -/* ver_psmodules, */ -/* ver_psphot, */ -/* ver_psastro, */ -/* ver_ppstats, */ -/* ver_ppimage, */ -/* ver_streaks */ + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { // rollback @@ -1370,4 +1395,32 @@ psMetadata *row = output->data[i]; + psS64 chip_id = psMetadataLookupS64(NULL,row,"chip_id"); + + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"chiptool_coalesce_run.sql",chip_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + psMetadataAddStr(row, PS_LIST_TAIL, "software_ver", 0, "Software version", software_ver); + psMetadataAddS32(row, PS_LIST_TAIL, "maskfrac_npix", 0, "Number of pixels used for mask stats", maskfrac_npix); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_static", 0, "Fraction of static masked pixels", maskfrac_static); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_dynamic", 0, "Fraction of dynamic masked pixels", maskfrac_dynamic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_magic", 0, "Fraction of magic masked pixels", maskfrac_magic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_advisory", 0, "Fraction of advisory pixels", maskfrac_advisory); + psWarning("ADVANCE %ld %s %d %f %f %f %f\n",chip_id,software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory); chipRunRow *chipRun = chipRunObjectFromMetadata(row); if (!psDBTransaction(config->dbh)) { @@ -1376,4 +1429,25 @@ } + // Set chipRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "chipRun", "chip_id", chip_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set chipRun.maskfrac* to the appropriate values. + if (!pxSetRunMaskfrac(config, "chipRun", "chip_id",chip_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } // set chipRun.state to 'stop' and update the magicked state if (!pxchipRunSetState(config, chipRun->chip_id, "full", chipRun->magicked)) { @@ -1400,16 +1474,16 @@ // stop if (!pxcamQueueByChipID(config, - chipRun->chip_id, - chipRun->workdir, - chipRun->label, - chipRun->data_group, - chipRun->dist_group, - chipRun->reduction, - chipRun->expgroup, - chipRun->dvodb, - chipRun->tess_id, - chipRun->end_stage, - chipRun->magicked, - NULL // note does not propagate + chipRun->chip_id, + chipRun->workdir, + chipRun->label, + chipRun->data_group, + chipRun->dist_group, + chipRun->reduction, + chipRun->expgroup, + chipRun->dvodb, + chipRun->tess_id, + chipRun->end_stage, + chipRun->magicked, + NULL // note does not propagate )) { if (!psDBRollback(config->dbh)) { Index: branches/czw_branch/20100427/ippTools/src/chiptoolConfig.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/chiptoolConfig.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/chiptoolConfig.c (revision 28017) @@ -187,7 +187,13 @@ psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); - psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_ppImage", 0, "define ppImage version", NULL); + psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_ppimage", 0, "define ppImage version", NULL); psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + psMetadataAddS32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + // -processedimfile psMetadata *processedimfileArgs = psMetadataAlloc(); Index: branches/czw_branch/20100427/ippTools/src/difftool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/difftool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/difftool.c (revision 28017) @@ -58,4 +58,5 @@ static bool tofullskyfileMode(pxConfig *config); static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); @@ -103,4 +104,5 @@ MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); MODECASE(DIFFTOOL_MODE_LISTRUN, listrunMode); + MODECASE(DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); default: @@ -160,4 +162,10 @@ exposure, false, + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory diff_mode, note @@ -555,7 +563,29 @@ PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s ppsub %s streaks %s\n", ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_ppsub,ver_streaks); - + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_ppsub) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppsub); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } + if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -590,13 +620,11 @@ fault, quality, - magicked -/* magicked, */ -/* ver_pslib, */ -/* ver_psmodules, */ -/* ver_psphot, */ -/* ver_ppstats, */ -/* ver_ppsub, */ -/* ver_streaks */ - + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { if (!psDBRollback(config->dbh)) { @@ -687,4 +715,44 @@ psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"difftool_coalesce_run.sql",diff_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set diffRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "diffRun", "diff_id", diff_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set diffRun.software_ver for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set diffRun.maskfrac* to the appropriate values. + if (!pxSetRunMaskfrac(config, "diffRun", "diff_id", diff_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set diffRun.software_ver for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } // set diffRun.state to 'full' if (!setdiffRunState(config, diff_id, "full", magicked)) { @@ -840,48 +908,13 @@ } - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Update state to 'new' - { - psString query = pxDataGet("difftool_revertdiffskyfile_update.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); - } - - // Delete product { psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } + return false; + } + psString query_updated = pxDataGet("difftool_revertdiffskyfile_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -890,4 +923,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); psFree(whereClause); } @@ -896,20 +930,21 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } return false; } psFree(query); - psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); + + psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); } psFree(where); - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } return true; @@ -1034,4 +1069,10 @@ false, 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory diff_mode, // diff_mode note @@ -1461,4 +1502,10 @@ true, // exposure 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_WARP_STACK, note @@ -1813,4 +1860,10 @@ true, // exposure false, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_WARP_WARP, note); // Run to insert @@ -2095,4 +2148,10 @@ false, // exposure 0, // magicked + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory IPP_DIFF_MODE_STACK_STACK, // diff_mode note @@ -2388,4 +2447,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) { @@ -2768,2 +2828,36 @@ return true; } +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("difftool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , diffRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} Index: branches/czw_branch/20100427/ippTools/src/difftool.h =================================================================== --- branches/czw_branch/20100427/ippTools/src/difftool.h (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/difftool.h (revision 28017) @@ -48,5 +48,6 @@ DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, DIFFTOOL_MODE_TOFULLSKYFILE, - DIFFTOOL_MODE_LISTRUN + DIFFTOOL_MODE_LISTRUN, + DIFFTOOL_MODE_SETSKYFILETOUPDATE, } difftoolMode; Index: branches/czw_branch/20100427/ippTools/src/difftoolConfig.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/difftoolConfig.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/difftoolConfig.c (revision 28017) @@ -132,5 +132,10 @@ psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_ppsub", 0, "define ppSub version", NULL); psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); - + psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + // -advance psMetadata *advanceArgs = psMetadataAlloc(); @@ -332,5 +337,6 @@ psMetadata *updatediffskyfileArgs = psMetadataAlloc(); psMetadataAddS64(updatediffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); - psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); // -tocleanedskyfile @@ -354,4 +360,12 @@ psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); + + // -setskyfiletoupdate + psMetadata *setskyfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setskyfiletoupdateArgs, PS_LIST_TAIL, "-diff_id", 0, "search by difftool ID (required)", 0); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "new value for diffRun.label", NULL); + + // -exportrun @@ -391,4 +405,6 @@ PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", DIFFTOOL_MODE_DONECLEANUP, donecleanupArgs); PXOPT_ADD_MODE("-updatediffskyfile", "update fault code for a diffskyfile", DIFFTOOL_MODE_UPDATEDIFFSKYFILE, updatediffskyfileArgs); + PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", DIFFTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", DIFFTOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/czw_branch/20100427/ippTools/src/magicdstool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/magicdstool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/magicdstool.c (revision 28017) @@ -572,5 +572,4 @@ // we can transition by query as well - psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); @@ -1225,11 +1224,4 @@ } - psString stateCheck = NULL; - if (!strcmp(state, "goto_cleaned")) { - // science cleanup sets runs to goto_cleaned. Don't undo it if already cleaned - psStringAppend(&stateCheck, " AND magicDSRun.state != 'cleaned'"); - } - - psString query = NULL; psStringAppend(&query, "UPDATE magicDSRun SET state = '%s' %s\n", state, extraSetStr ? extraSetStr : ""); @@ -1241,12 +1233,16 @@ psFree(whereClause); } else { - psError(PS_ERR_UNKNOWN, true, "search arugments are required"); - return false; - } - - if (stateCheck) { - psStringAppend(&query, stateCheck); - } - psFree(stateCheck); + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; + } + + if (!strcmp(state, "goto_cleaned")) { + // Don't set state back to goto_cleaned if it is already cleaned + psStringAppend(&query, " AND (magicDSRun.state != 'cleaned')"); + + // don't clean up magicDSRun's where stage is camera + psStringAppend(&query, " AND (magicDSRun.stage != 'camera')"); + } + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, Index: branches/czw_branch/20100427/ippTools/src/magictool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/magictool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/magictool.c (revision 28017) @@ -48,4 +48,5 @@ static bool maskMode(pxConfig *config); static bool censorrunMode(pxConfig *config); +static bool exposureMode(pxConfig *config); static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -87,4 +88,5 @@ MODECASE(MAGICTOOL_MODE_MASK, maskMode); MODECASE(MAGICTOOL_MODE_CENSORRUN, censorrunMode); + MODECASE(MAGICTOOL_MODE_EXPOSURE, exposureMode); default: psAbort("invalid option (this should not happen)"); @@ -1514,2 +1516,66 @@ return true; } + + +static bool exposureMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_BOOL(config->args, where, "-inverse", "magicRun.inverse", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_exposure.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psStringCopy(""); // WHERE restrictions + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\nWHERE %s", clause); + psFree(clause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "magicMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} Index: branches/czw_branch/20100427/ippTools/src/magictool.h =================================================================== --- branches/czw_branch/20100427/ippTools/src/magictool.h (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/magictool.h (revision 28017) @@ -42,4 +42,5 @@ MAGICTOOL_MODE_MASK, MAGICTOOL_MODE_CENSORRUN, + MAGICTOOL_MODE_EXPOSURE, } MAGICtoolMode; Index: branches/czw_branch/20100427/ippTools/src/magictoolConfig.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/magictoolConfig.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/magictoolConfig.c (revision 28017) @@ -168,4 +168,10 @@ psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); psMetadataAddStr(censorrunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + + // -exposure + psMetadata *exposureArgs = psMetadataAlloc(); + psMetadataAddS64(exposureArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); + psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-inverse", 0, "select the inverse subtraction?", false); + psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -191,4 +197,5 @@ PXOPT_ADD_MODE("-mask", "", MAGICTOOL_MODE_MASK, maskArgs); PXOPT_ADD_MODE("-censorrun", "", MAGICTOOL_MODE_CENSORRUN, censorrunArgs); + PXOPT_ADD_MODE("-exposure", "", MAGICTOOL_MODE_EXPOSURE, exposureArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/czw_branch/20100427/ippTools/src/pstamptool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/pstamptool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/pstamptool.c (revision 28017) @@ -40,4 +40,5 @@ static bool updatereqMode(pxConfig *config); static bool revertreqMode(pxConfig *config); +static bool pendingcleanupMode(pxConfig *config); static bool addjobMode(pxConfig *config); static bool listjobMode(pxConfig *config); @@ -81,4 +82,5 @@ MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updatereqMode); MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertreqMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGCLEANUP, pendingcleanupMode); MODECASE(PSTAMPTOOL_MODE_ADDJOB, addjobMode); MODECASE(PSTAMPTOOL_MODE_LISTJOB, listjobMode); @@ -265,4 +267,5 @@ NULL, // outProduct uri, + NULL, // outdir 0 // fault )) { @@ -301,4 +304,6 @@ } psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); // treat limit == 0 as "no limit" @@ -339,5 +344,5 @@ } -static bool listreqMode(pxConfig *config) +static bool pendingcleanupMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -345,21 +350,23 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); - PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - if (!psListLength(where->list)) { - psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); - return false; - } - - psString query = psStringCopy("SELECT * from pstampRequest"); - - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - psFree(where); + psString query = pxDataGet("pstamptool_pendingcleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampRequest"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); // treat limit == 0 as "no limit" @@ -383,10 +390,7 @@ } if (!psArrayLength(output)) { - psTrace("pstamptool", PS_LOG_INFO, "request not found"); - // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for - psError(PXTOOLS_ERR_CONFIG, true, "request not found"); - psFree(output); - // we return false so that the caller can determine that a request does not exist - return false; + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; } @@ -403,21 +407,26 @@ } -static bool completedreqMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); +static bool listreqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = pxDataGet("pstamptool_completedreq.sql"); - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } + if (!psListLength(where->list)) { + psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); + return false; + } + + psString query = psStringCopy("SELECT * from pstampRequest"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); psFree(where); @@ -434,4 +443,5 @@ return false; } + psFree(query); psArray *output = p_psDBFetchResult(config->dbh); @@ -440,9 +450,11 @@ return false; } - if (!psArrayLength(output)) { - psTrace("pstamptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; + psTrace("pstamptool", PS_LOG_INFO, "request not found"); + // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for + psError(PXTOOLS_ERR_CONFIG, true, "request not found"); + psFree(output); + // we return false so that the caller can determine that a request does not exist + return false; } @@ -459,23 +471,103 @@ } +static bool completedreqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_completedreq.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + static bool updatereqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); - PXOPT_LOOKUP_STR(outProduct, config->args, "-outProduct", false, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); - PXOPT_LOOKUP_STR(reqType, config->args, "-reqType", false, false); - - psString query = NULL; - psStringAppend(&query, "UPDATE pstampRequest SET"); - + // PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-set_outProduct", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-set_uri", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-set_name", false, false); + PXOPT_LOOKUP_STR(reqType, config->args, "-set_reqType", false, false); + PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id_max", "req_id", "<="); + PXOPT_COPY_S32(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE pstampRequest SET"); + + psString stateCheck = NULL; char c = ' '; if (state) { psStringAppend(&query, "%c state = '%s'", c, state); c = ','; + if (!strcmp(state, "goto_cleaned")) { + psStringAppend(&stateCheck, " AND state != 'cleaned'"); + } + } + if (label) { + psStringAppend(&query, "%c label = '%s'", c, label); + c = ','; } if (outProduct) { @@ -483,5 +575,16 @@ c = ','; } - if (fault) { + if (outdir) { + psStringAppend(&query, "%c outdir = '%s'", c, outdir); + c = ','; + } + if (clearfault) { + if (fault) { + psError(PXTOOLS_ERR_CONFIG, true, "only one of -fault and -clearfault is allowed"); + return false; + } + psStringAppend(&query, "%c fault = 0", c); + c = ','; + } else if (fault) { psStringAppend(&query, "%c fault = %d", c, fault); c = ','; @@ -499,10 +602,17 @@ c = ','; } - if (!state && !outProduct && !fault && !uri && !name && !reqType) { - psError(PS_ERR_UNKNOWN, true, "at least one of state, outProduct, fault, uri, name, and reqType must be specified"); - return false; - } - - psStringAppend(&query, " WHERE req_id = %" PRId64, req_id); + if (c != ',') { + psError(PS_ERR_UNKNOWN, true, "at least one set option is required"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (stateCheck) { + psStringAppend(&query, "%s", stateCheck); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -512,11 +622,6 @@ } - psU64 affected = psDBAffectedRows(config->dbh); - // note zero is not an error - if (affected > 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" - PRIu64 " rows were modified", affected); - return false; - } + // psU64 affected = psDBAffectedRows(config->dbh); + // psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampRequests", affected); return true; @@ -530,5 +635,5 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "=="); pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); @@ -705,5 +810,5 @@ PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -724,4 +829,6 @@ psFree(where); + psStringAppend(&query, " ORDER BY priority DESC, req_id"); + // treat limit == 0 as "no limit" if (limit) { @@ -772,6 +879,12 @@ return false; } - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_S32(fault, config->args, "-set_fault", false, false); + + if (!state && !fault) { + psError(PS_ERR_UNKNOWN, true, "at least one of -set_state and -set_fault is required"); + return false; + } psMetadata *where = psMetadataAlloc(); @@ -780,13 +893,16 @@ PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); - psString faultStr = NULL; - if (!fault) { - faultStr = psStringCopy(""); - } else { - psStringAppend(&faultStr, "\n, pstampJob.fault = %d", fault); - } - psString query = pxDataGet("pstamptool_updatejob.sql"); + char c = ' '; + if (state) { + psStringAppend(&query, "\n %c pstampJob.state = '%s'", c, state); + c = ','; + } + if (fault) { + psStringAppend(&query, "\n %c pstampJob.fault = %d", c, fault); + c = ','; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " WHERE %s", whereClause); @@ -794,10 +910,9 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, state, faultStr)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); return false; } - psFree(faultStr); psFree(query); @@ -967,5 +1082,6 @@ PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); PXOPT_LOOKUP_STR(imagedb, config->args, "-imagedb", true, false); - PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", false, false); + PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", true, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", true, false); PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false); PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); @@ -983,5 +1099,5 @@ } - psString query = pxDataGet("pstamptool_pendingdependent.sql"); + psString query = pxDataGet("pstamptool_getdependent.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -1046,4 +1162,5 @@ rlabel, need_magic, + outdir, 0 // fault )) { @@ -1087,4 +1204,5 @@ PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "=="); PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); PXOPT_COPY_STR(config->args, where, "-rlabel", "rlabel", "=="); pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); @@ -1105,4 +1223,6 @@ } psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); // treat limit == 0 as "no limit" @@ -1148,5 +1268,5 @@ PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", true, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); if (!fault && !state) { @@ -1186,5 +1306,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "pstampDependent.fault", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); @@ -1201,6 +1321,6 @@ } - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); psFree(where); Index: branches/czw_branch/20100427/ippTools/src/pstamptool.h =================================================================== --- branches/czw_branch/20100427/ippTools/src/pstamptool.h (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/pstamptool.h (revision 28017) @@ -34,4 +34,5 @@ PSTAMPTOOL_MODE_UPDATEREQ, PSTAMPTOOL_MODE_REVERTREQ, + PSTAMPTOOL_MODE_PENDINGCLEANUP, PSTAMPTOOL_MODE_ADDJOB, PSTAMPTOOL_MODE_LISTJOB, Index: branches/czw_branch/20100427/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/pstamptoolConfig.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/pstamptoolConfig.c (revision 28017) @@ -98,11 +98,18 @@ // -updatereq psMetadata *updatereqArgs = psMetadataAlloc(); - psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); - psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", 0); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to update", 0); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id_max", 0, "maximum req_id for which to update", 0); + psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_state", 0, "new state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_label", 0, "new label", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_outProduct", 0, "new outProduct", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_outdir", 0, "new outdir", NULL); + psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-set_fault", 0, "new fault code", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_uri", 0, "new uri", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_name", 0, "new name", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_reqType", 0, "new reqType", NULL); + psMetadataAddBool(updatereqArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); // -revertreq @@ -112,4 +119,12 @@ psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); + + // -pendingcleanup + psMetadata *pendingcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddStr(pendingcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); + psMetadataAddU64(pendingcleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -addjob @@ -145,6 +160,6 @@ psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id for which to change state", 0); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", 0); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0, "new state", NULL); + psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0, "new result", 0); // -revertjob @@ -160,5 +175,7 @@ // -getdependent psMetadata *getdependentArgs = psMetadataAlloc(); - psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent (required)", 0); + // arguabely all of these could be -set_ arguments, but since this mode is both a query and + // a create if doesn't exist I like it this way + psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage id for dependent (required)", 0); psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent (required)", NULL); psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for depenent (required)", NULL); @@ -166,4 +183,5 @@ psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent ", NULL); psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-need_magic", 0, "define need_magic", false); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory for dependent (required)", NULL); psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no matching dependent do not create one", false); @@ -172,10 +190,11 @@ psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent (required)", 0); psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); - psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-fault", 0, "new value for fault", 0); + psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-set_fault", 0, "new value for fault", 0); // -revertdependent psMetadata *revertdependentArgs = psMetadataAlloc(); psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent", 0); - psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault", 0, "new value for fault", 0); + psMetadataAddStr(revertdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for dependent ", NULL); + psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault", 0, "define fault", 0); // -pendingdependent @@ -184,4 +203,5 @@ psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent", NULL); psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for dependent", NULL); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define dep_id for dependent", 0); psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for pstampRequest", NULL); psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent", NULL); @@ -224,4 +244,5 @@ PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); PXOPT_ADD_MODE("-revertreq", "", PSTAMPTOOL_MODE_REVERTREQ, revertreqArgs); + PXOPT_ADD_MODE("-pendingcleanup", "", PSTAMPTOOL_MODE_PENDINGCLEANUP, pendingcleanupArgs); PXOPT_ADD_MODE("-addjob", "", PSTAMPTOOL_MODE_ADDJOB, addjobArgs); Index: branches/czw_branch/20100427/ippTools/src/pxcam.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/pxcam.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/pxcam.c (revision 28017) @@ -152,5 +152,5 @@ char *tess_id, char *end_stage, - psS64 magicked, + psS64 magicked, char *note) { @@ -184,4 +184,10 @@ end_stage ? end_stage : "NULL", (long long) magicked, + "NULL", // This shouldn't be propagated + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", note ? note : "NULL", (long long)chip_id Index: branches/czw_branch/20100427/ippTools/src/pxchip.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/pxchip.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/pxchip.c (revision 28017) @@ -69,9 +69,9 @@ psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); - psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); return true; } @@ -121,5 +121,5 @@ PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "LIKE"); PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); @@ -277,4 +277,10 @@ end_stage, 0, // magicked + NULL, + 0, + NAN, + NAN, + NAN, + NAN, note ) Index: branches/czw_branch/20100427/ippTools/src/pxtools.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/pxtools.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/pxtools.c (revision 28017) @@ -57,4 +57,13 @@ psS32 num2; psS32 numO; + + if (!version1) { + psStringAppend(&out,version2); + return(out); + } + if (!version2) { + psStringAppend(&out,version1); + return(out); + } if (strchr(version1,'M')) { @@ -84,5 +93,73 @@ } - +bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, + psS32 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psF32 *maskfrac_magic, psF32 *maskfrac_advisory) { + psString query = pxDataGet(dbQFile); +/* psString text_id = NULL; */ +/* psStringAppend(&text_id," %" PRId64,stage_id); */ +/* psStringSubstitute(&query,text_id,"@STAGE_ID@"); */ +/* psFree(text_id); */ + if (!p_psDBRunQueryF(config->dbh, query, stage_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS32 this_npix = psMetadataLookupS32(NULL, row, "maskfrac_npix"); + psF32 this_static = psMetadataLookupF32(NULL, row, "maskfrac_static"); + psF32 this_dynamic = psMetadataLookupF32(NULL, row, "maskfrac_dynamic"); + psF32 this_magic = psMetadataLookupF32(NULL, row, "maskfrac_magic"); + psF32 this_advisory = psMetadataLookupF32(NULL, row, "maskfrac_advisory"); + psString this_version = psMetadataLookupStr(NULL, row, "software_ver"); + + *software_ver = pxMergeCodeVersions(*software_ver,this_version); + + *maskfrac_static = ((*maskfrac_static * *maskfrac_npix) + (this_npix * this_static)) / (this_npix + *maskfrac_npix); + *maskfrac_dynamic = ((*maskfrac_dynamic * *maskfrac_npix) + (this_npix * this_dynamic)) / (this_npix + *maskfrac_npix); + *maskfrac_magic = ((*maskfrac_magic * *maskfrac_npix) + (this_npix * this_magic)) / (this_npix + *maskfrac_npix); + *maskfrac_advisory = ((*maskfrac_advisory * *maskfrac_npix) + (this_npix * this_advisory)) / (this_npix + *maskfrac_npix); + *maskfrac_npix += this_npix; + } + psFree(output); + return(true); +} + +bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psString software_ver) { + char *query = "UPDATE %s SET software_ver = '%s' WHERE %s = %" PRId64; + printf(query,tableName,software_ver,stage_id_name,stage_id); + if (!p_psDBRunQueryF(config->dbh,query,tableName,software_ver,stage_id_name,stage_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set software version for %s %" PRId64,stage_id_name,stage_id); + return(false); + } + + return(true); +} +bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psS32 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psF32 maskfrac_magic, psF32 maskfrac_advisory) { + char *query = "UPDATE %s SET maskfrac_npix = %d, maskfrac_static = %f, maskfrac_dynamic = %f, maskfrac_magic = %f, maskfrac_advisory = %f WHERE %s = %" PRId64; + if (!p_psDBRunQueryF(config->dbh,query,tableName,maskfrac_npix,maskfrac_static, + maskfrac_dynamic, maskfrac_magic,maskfrac_advisory,stage_id_name,stage_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set maskfrac stats for %s %" PRId64,stage_id_name,stage_id); + return(false); + } + + + + return(true); +} // 'scrubbed' is no longer a virtual state equivalent to cleaned, but allows files to be removed Index: branches/czw_branch/20100427/ippTools/src/pxtools.h =================================================================== --- branches/czw_branch/20100427/ippTools/src/pxtools.h (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/pxtools.h (revision 28017) @@ -53,4 +53,14 @@ bool pxIsValidState(const char *state); +psString pxMergeCodeVersions(psString version1, psString version2); +bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, + psS32 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psF32 *maskfrac_magic, psF32 *maskfrac_advisory); +bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psString software_ver); +bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psS32 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psF32 maskfrac_magic, psF32 maskfrac_advisory); + bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows); bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *field, char *name, char *op); @@ -286,4 +296,6 @@ PXOPT_COPY_PRIMITIVE(from, to, psU64, U64, oldname, newname, comment) +#define PXOPT_COPY_BOOL(from, to, oldname, newname, comment) \ + PXOPT_COPY_V(from, to, psBool, Bool, oldname, newname, comment) /*** these PXOPT_ADD_WHERE macros are used to construct the default sql elements ***/ Index: branches/czw_branch/20100427/ippTools/src/pxwarp.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/pxwarp.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/pxwarp.c (revision 28017) @@ -60,4 +60,10 @@ NULL, // registered 0, // magicked set to zero when created may get updated when warpRun goes to 'full' + NULL, // version + 0, // maskfrac_npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory NULL // note )) { Index: branches/czw_branch/20100427/ippTools/src/stacktool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/stacktool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/stacktool.c (revision 28017) @@ -370,4 +370,5 @@ tess_id, filter, + NULL, // software_ver note); @@ -526,4 +527,5 @@ tess_id, filter, + NULL, // software_ver note); @@ -853,4 +855,20 @@ psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s ppstack %s streaks %s\n", ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_ppstack,ver_streaks); + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + if (ver_ppstack) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstack); + } + if (ver_streaks) { + software_ver = pxMergeCodeVersions(software_ver,ver_streaks); + } // default values @@ -893,12 +911,6 @@ good_frac, fault, + software_ver, quality -/* quality, */ -/* ver_pslib, */ -/* ver_psmodules, */ -/* ver_psphot, */ -/* ver_ppstats, */ -/* ver_ppstack, */ -/* ver_streaks */ )) { if (!psDBRollback(config->dbh)) { @@ -910,4 +922,14 @@ if (fault == 0) { + // Set stackRun software if we are finished. + if (!pxSetRunSoftware(config, "stackRun", "stack_id", stack_id, software_ver)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set stackRun.software_ver for stack_id: %" PRId64, + stack_id); + return(false); + } + if (!setstackRunState(config, stack_id, "full")) { if (!psDBRollback(config->dbh)) { Index: branches/czw_branch/20100427/ippTools/src/staticskytool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/staticskytool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/staticskytool.c (revision 28017) @@ -107,9 +107,9 @@ // optional - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); - PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); @@ -118,10 +118,10 @@ PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); - PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "stackRun.skycell_id", "=="); - pxAddLabelSearchArgs(config, where, "-select_label", "stackRun.label", "LIKE"); - pxAddLabelSearchArgs(config, where, "-select_filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "stackRun.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-select_filter", "stackRun.filter", "LIKE"); // these are used to build the HAVING restriction - PXOPT_COPY_S32(config->args, having, "-min_num", "num", ">="); + PXOPT_COPY_S32(config->args, having, "-min_num", "num", ">="); PXOPT_COPY_S32(config->args, having, "-min_filter", "num_filter", ">="); Index: branches/czw_branch/20100427/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/staticskytoolConfig.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/staticskytoolConfig.c (revision 28017) @@ -46,17 +46,18 @@ // -definebyquery + // XXX need a 'ra_min,max', 'dec_min,max' : to do this, we need a table of skycell boundaries psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -64,5 +65,5 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ss_id", 0, "search by stack ID", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by stack ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); @@ -75,5 +76,5 @@ // -inputs psMetadata *inputsArgs = psMetadataAlloc(); - psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -81,5 +82,5 @@ // -todo psMetadata *todoArgs = psMetadataAlloc(); - psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -88,5 +89,5 @@ // -addresult psMetadata *addresultArgs = psMetadataAlloc(); - psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ss_id", 0, "define stack ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-sky_id", 0, "define stack ID (required)", 0); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN); @@ -100,5 +101,5 @@ // -result psMetadata *resultArgs= psMetadataAlloc(); - psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); @@ -111,5 +112,5 @@ // -revert psMetadata *revertArgs= psMetadataAlloc(); - psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -118,10 +119,10 @@ // -updateresult psMetadata *updateresultArgs = psMetadataAlloc(); - psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ss_id", 0, "define staticksky ID (required)", 0); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-sky_id", 0, "define staticksky ID (required)", 0); psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); - psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ss_id", 0, "export this staticsky ID (required)", 0); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-sky_id", 0, "export this staticsky ID (required)", 0); psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/czw_branch/20100427/ippTools/src/warptool.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/warptool.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/warptool.c (revision 28017) @@ -177,5 +177,11 @@ registered, 0, // magicked - note + NULL, // software version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory + note ); if (!warpRun) { @@ -248,4 +254,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); @@ -273,4 +281,5 @@ PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); // check mode @@ -310,4 +319,15 @@ if (!psArrayLength(output)) { psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } psFree(output); return true; @@ -925,5 +945,5 @@ // to avoid slow queries and to the query itself // to satisfy the user's requested limit - psStringAppend(&limitString, psDBGenerateLimitSQL(limit)); + psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); psStringAppend(&query, " %s", limitString); } @@ -1012,7 +1032,28 @@ PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s pswarp %s streaks %s\n", ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_pswarp,ver_streaks); - + psString ver_code = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + ver_code = pxMergeCodeVersions(ver_code,ver_psphot); + } + if (ver_ppstats) { + ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); + } + if (ver_pswarp) { + ver_code = pxMergeCodeVersions(ver_code,ver_pswarp); + } + if (ver_streaks) { + ver_code = pxMergeCodeVersions(ver_code,ver_streaks); + } // default values @@ -1049,12 +1090,11 @@ fault, quality, - magicked -/* magicked, */ -/* ver_pslib, */ -/* ver_psmodules, */ -/* ver_psphot, */ -/* ver_ppstats, */ -/* ver_pswarp, */ -/* ver_streaks */ + magicked, + ver_code, + maskfrac_npix, + maskfrac_static, + maskfrac_dynamic, + maskfrac_magic, + maskfrac_advisory )) { if (!psDBRollback(config->dbh)) { @@ -1121,4 +1161,47 @@ bool status; psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); + + psString software_ver = NULL; + psS32 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"warptool_coalesce_run.sql",warp_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set warpRun.software_ver to the appropriate value + if (!pxSetRunSoftware(config, "warpRun", "warp_id", warp_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, + warp_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + // Set warpRun.maskfrac* to the appropriate values. + if (!pxSetRunMaskfrac(config, "warpRun", "warp_id", warp_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, + warp_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + + if (!status) { psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id"); Index: branches/czw_branch/20100427/ippTools/src/warptoolConfig.c =================================================================== --- branches/czw_branch/20100427/ippTools/src/warptoolConfig.c (revision 27784) +++ branches/czw_branch/20100427/ippTools/src/warptoolConfig.c (revision 28017) @@ -89,6 +89,8 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by fakeRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on fakeRun label", NULL); @@ -106,4 +108,5 @@ psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); // -definerun @@ -227,4 +230,10 @@ psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_pswarp", 0, "define pswarp version", NULL); psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + psMetadataAddS32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN);