Index: branches/eam_branches/20091201/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/20091201/ippTools/src/Makefile.am (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/Makefile.am (revision 26857) @@ -9,4 +9,5 @@ disttool \ flatcorr \ + dqstatstool \ magictool \ magicdstool \ @@ -32,4 +33,5 @@ pxdata.h \ pxfake.h \ + pxdqstats.h pxmagic.h \ pxregister.h \ @@ -52,4 +54,5 @@ flatcorr.h \ faketool.h \ + dqstatstool.h \ magictool.h \ magicdstool.h \ @@ -77,4 +80,5 @@ pxfake.c \ pxfault.c \ + pxdqstats.c \ pxmagic.c \ pxregister.c \ @@ -142,4 +146,10 @@ faketool.c \ faketoolConfig.c + +dqstatstool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +dqstatstool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +dqstatstool_SOURCES = \ + dqstatstool.c \ + dqstatstoolConfig.c magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: branches/eam_branches/20091201/ippTools/src/camtool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/camtool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/camtool.c (revision 26857) @@ -222,4 +222,5 @@ tess_id ? tess_id : row->tess_id, end_stage ? end_stage : row->end_stage, + row->magicked, note )) { @@ -480,5 +481,6 @@ PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); +// Get this from the chipRun +// PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); // generate restrictions @@ -523,4 +525,7 @@ return false; } + + bool status; + psS64 chip_magicked = psMetadataLookupS64(&status, output->data[0], "chip_magicked"); camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]); @@ -603,8 +608,5 @@ } - // NULL for end_stage means go as far as possible - // EAM : skip here if fault != 0 - // Also, we can run fake even if tess_id is not defined - if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) { + if (fault) { psFree(row); psFree(pendingRow); @@ -618,5 +620,6 @@ // since there is only one exp per 'new' set camRun.state = 'full' - if (!pxcamRunSetState(config, row->cam_id, "full", magicked)) { + // propagate magicked state from chipRun + if (!pxcamRunSetState(config, row->cam_id, "full", chip_magicked)) { psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); psFree(row); @@ -625,4 +628,15 @@ } psFree(row); + + // EAM: NULL for end_stage means go as far as possible + // Also, we can run fake even if tess_id is not defined + if (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam")) { + psFree(pendingRow); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } if (!pxfakeQueueByCamID(config, Index: branches/eam_branches/20091201/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/chiptool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/chiptool.c (revision 26857) @@ -612,5 +612,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); psFree(whereClause); } else if (psMetadataLookupBool(NULL, config->args, "-all") || (faulted || where2)) { @@ -720,4 +720,8 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); + if (psListLength(where->list) == 0) { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } if (!state) { @@ -1254,4 +1258,5 @@ chipRun->tess_id, chipRun->end_stage, + chipRun->magicked, NULL // note does not propagate )) { @@ -1295,13 +1300,20 @@ } - psString set_magic = ""; + char *set_magicked_imfile = ""; + char *set_magicked_run = ""; if (!strcmp(data_state, "full")) { - // copy the magicked state from the input to the output when transitioning to full state - set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked"; - } - - // note only updates if chipRun.state = run_state - // XXX note that we have removed this constraint for now - if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) { + // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will + // block warp processing until destreaking has been done + // otherwise copy magicked from the rawImfile + // Same thing for chipRun/rawExp + set_magicked_imfile = "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0 AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)"; + set_magicked_run = "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), chipRun.magicked, rawExp.magicked)"; + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + // if magicked is non-zero set it to -1 + set_magicked_imfile = "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)"; + set_magicked_run = "\n, chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"; + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) { psFree(query); psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1320,5 +1332,5 @@ query = pxDataGet("chiptool_change_exp_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) { psFree(query); // rollback Index: branches/eam_branches/20091201/ippTools/src/chiptoolConfig.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/chiptoolConfig.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/chiptoolConfig.c (revision 26857) @@ -154,5 +154,5 @@ psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); - psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", false); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", 0); // -processedimfile Index: branches/eam_branches/20091201/ippTools/src/dettool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/dettool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/dettool.c (revision 26857) @@ -550,4 +550,5 @@ PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "airmass", "<="); PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_max", "sat_pixel_frac", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_min", "sat_pixel_frac", ">="); PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "exp_time", ">="); PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "exp_time", "<="); Index: branches/eam_branches/20091201/ippTools/src/dettoolConfig.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/dettoolConfig.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/dettoolConfig.c (revision 26857) @@ -117,4 +117,5 @@ psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, "define max airmass", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_min", 0, "define min fraction of saturated pixels", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, "define min exposure time", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, "define max exposure time", NAN); Index: branches/eam_branches/20091201/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/difftool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/difftool.c (revision 26857) @@ -55,4 +55,5 @@ static bool topurgedskyfileMode(pxConfig *config); static bool toscrubbedskyfileMode(pxConfig *config); +static bool tofullskyfileMode(pxConfig *config); @@ -94,7 +95,8 @@ MODECASE(DIFFTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(DIFFTOOL_MODE_IMPORTRUN, importrunMode); - MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); - MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); - MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); default: @@ -186,5 +188,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-diff_id", "stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); @@ -1146,6 +1148,8 @@ PXOPT_COPY_STR(config->args, warp1Where, "-warp_label", "warpRun.label", "=="); PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "=="); PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "=="); PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "=="); PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "=="); @@ -1361,4 +1365,6 @@ return false; } + + psString filter = psMetadataLookupStr(&mdok, row, "filter"); if (!mdok) { @@ -2036,4 +2042,11 @@ PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + psS64 magicked = 0; + if (!strcmp(data_state, "full")) { + PXOPT_LOOKUP_S64(set_magicked, config->args, "-magicked", 0, false); + magicked = set_magicked; + } + psString query = pxDataGet("difftool_change_skyfile_data_state.sql"); @@ -2043,7 +2056,18 @@ } + psString set_magicked_skyfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (magicked != 0 && !strcmp(data_state, "full")) { + psStringAppend(&set_magicked_skyfile, "\n , diffSkyfile.magicked = %" PRId64, magicked); + psStringAppend(&set_magicked_run, "\n , diffRun.magicked = %" PRId64, magicked); + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + psStringAppend(&set_magicked_skyfile, "\n, diffSkyfile.magicked = IF(diffSkyfile.magicked = 0, 0, -1)"); + psStringAppend(&set_magicked_run, "\n, diffRun.magicked = IF(diffRun.magicked = 0, 0, -1)"); + } + // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c - if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, skycell_id)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, diff_id, skycell_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); // rollback @@ -2052,10 +2076,12 @@ } psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_skyfile); return(false); } + psFree(set_magicked_skyfile); psFree(query); query = pxDataGet("difftool_change_run_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, diff_id, data_state)) { // rollback if (!psDBRollback(config->dbh)) { @@ -2063,4 +2089,5 @@ } psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_run); return(false); } @@ -2068,6 +2095,8 @@ if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_run); return(false); } + psFree(set_magicked_run); return(true); @@ -2082,4 +2111,7 @@ static bool toscrubbedskyfileMode(pxConfig *config) { return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); +} +static bool tofullskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "full", "update"); } Index: branches/eam_branches/20091201/ippTools/src/difftool.h =================================================================== --- branches/eam_branches/20091201/ippTools/src/difftool.h (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/difftool.h (revision 26857) @@ -45,5 +45,6 @@ DIFFTOOL_MODE_TOCLEANEDSKYFILE, DIFFTOOL_MODE_TOPURGEDSKYFILE, - DIFFTOOL_MODE_TOSCRUBBEDSKYFILE + DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, + DIFFTOOL_MODE_TOFULLSKYFILE } difftoolMode; Index: branches/eam_branches/20091201/ippTools/src/difftoolConfig.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/difftoolConfig.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/difftoolConfig.c (revision 26857) @@ -122,5 +122,5 @@ psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); - psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", false); + psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); // -advance @@ -183,4 +183,5 @@ psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); @@ -253,5 +254,5 @@ // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); - psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list runs for cleanup with specified label", NULL); psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -288,4 +289,10 @@ psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -tofullskyfile + psMetadata *tofullskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); // -exportrun @@ -329,4 +336,5 @@ PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); + PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20091201/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/disttool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/disttool.c (revision 26857) @@ -358,4 +358,5 @@ psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); psString stage_label = psMetadataLookupStr(NULL, md, "label"); + psString data_group = psMetadataLookupStr(NULL, md, "data_group"); bool clean = psMetadataLookupBool(NULL, md, "clean"); @@ -383,4 +384,5 @@ NULL, // time_stamp 0, // fault + data_group, set_note // note does not propagate ); @@ -1615,4 +1617,5 @@ PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); @@ -1662,5 +1665,5 @@ PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "rcInterest.state", "=="); PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); Index: branches/eam_branches/20091201/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/disttoolConfig.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/disttoolConfig.c (revision 26857) @@ -293,4 +293,5 @@ psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-dest_name", 0, "define destination name", NULL); Index: branches/eam_branches/20091201/ippTools/src/dqstatstool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/dqstatstool.c (revision 26857) +++ branches/eam_branches/20091201/ippTools/src/dqstatstool.c (revision 26857) @@ -0,0 +1,675 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdqstats.h" +#include "dqstatstool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool createbundleMode(pxConfig *config); +//static bool cleanbundleMode(pxConfig *config); +static bool pendingbundleMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool revertrunMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = dqstatstoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DQSTATS_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DQSTATS_MODE_PENDINGBUNDLE, pendingbundleMode); + MODECASE(DQSTATS_MODE_CREATEBUNDLE, createbundleMode); + // MODECASE(DQSTATS_MODE_CLEANBUNDLE, cleanbundleMode); + MODECASE(DQSTATS_MODE_UPDATERUN, updaterunMode); + MODECASE(DQSTATS_MODE_REVERTRUN, revertrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(EXIT_SUCCESS); + + FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exit_status); +} + +static bool definebyqueryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,NULL); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(force, config->args, "-force", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, 0); + + // properties of the raw exposures + psMetadata *where = psMetadataAlloc(); + + pxdqstatsGetSearchArgs(config, where); + + psString query = pxDataGet("dqstatstool_definebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return(false); + } + + if (label) { + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + } + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (force) { + psStringAppend(&query, " OR invalid = 0"); + } + + psFree(where); + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + // printf("%s\n",query); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dqstatstool", PS_LOG_INFO, "no rows found"); + psWarning("dqstatstool: no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "dqstatsContent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + // Create the run entry and grab the dqstats_id + dqstatsRunRow *run = dqstatsRunRowAlloc( + 0, // ID + "new", // state + registered, + set_label, + 0 // fault + ); + + if (!dqstatsRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + return(false); + } + + psS64 dqstats_id = psDBLastInsertID(config->dbh); + + psArray *list = psArrayAllocEmpty(16); // List of runs to print + psArrayAdd(list, list->n, run); + if (!dqstatsRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return(false); + } + psFree(list); + + // Create the content entry. + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + psS64 chip_id= psMetadataLookupS64(&status, row, "chip_id"); + psS64 cam_id = psMetadataLookupS64(&status, row, "cam_id"); + psS64 warp_id= psMetadataLookupS64(&status, row, "warp_id"); + + if (!dqstatsContentInsert(config->dbh, + dqstats_id, + exp_id, + chip_id, + cam_id, + warp_id, + 0 // do not create automatically invalid contents + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + return(true); +} + +static bool pendingbundleMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + + psMetadata *where = psMetadataAlloc(); + // pxdqstatsGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstatsRun.dqstats_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "dqstatsRun.label", "=="); + + psString query = pxDataGet("dqstatstool_get_run.sql"); // query + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retrieve sql statement (dqstatstool_get_run.sql)"); + return(false); + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); // where whereClause + + psArray *output = p_psDBFetchResult(config->dbh); // where whereClause output + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("dqstatstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "dqstatsPendingBundle", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return(true); +} +//static bool cleanbundleMode(pxConfig *config) { +// return(true); +//} + + +static bool createbundleMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + bool status = true; + PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", true, 0); + PXOPT_LOOKUP_STR(bundleUri, config->args, "-uri", true, 0); + + // Determine the information about this run. + psString runQuery = pxDataGet("dqstatstool_get_run.sql"); + if (!runQuery) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retrieve sql statement (dqstatstool_get_run.sql)"); + return(false); + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); // Save for content query + psStringAppend(&runQuery, " AND %s", whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, runQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(runQuery); + psFree(whereClause); + return(false); + } + psFree(runQuery); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + return(false); + } + + if (psArrayLength(output) != 1) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool: single return row not found."); + psFree(output); + psFree(whereClause); + return(false); + } + + psMetadata *run = output->data[0]; + if (strcmp(psMetadataLookupStr(&status,run,"state"),"new") != 0) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool: bundle already created for this run (should not happen)."); + psFree(output); + psFree(whereClause); + return(false); + } + psFree(output); // This held the information about the Run. + + // Read column list from config file. + psMetadata *recipe = psMetadataLookupPtr(&status, config->modules->recipes, "DQSTATS"); + psMetadata *columns = psMetadataLookupMetadata(&status,recipe,"STATS.COLUMNS"); + if (!columns) { + psError(PS_ERR_UNKNOWN, false, "Could not read STATS.COLUMN metadata"); + psFree(whereClause); + return(false); + } + + psMetadataIterator *iter = psMetadataIteratorAlloc(columns, PS_LIST_HEAD,NULL); + + psMetadataItem *colItem = NULL; + psString columnlist = NULL; + + psMetadata *colNames = psMetadataAlloc(); + psMetadata *colTypes = psMetadataAlloc(); + while ((colItem = psMetadataGetAndIncrement(iter))) { + bool status; + psMetadata *coldef = colItem->data.md; + + if (columnlist) { // Stick intervening commas + psStringAppend(&columnlist, ", "); + } + + psStringAppend(&columnlist, "%s.%s", + psMetadataLookupStr(&status,coldef,"TABLE"), + psMetadataLookupStr(&status,coldef,"ENTRY")); + + psMetadataAddStr(colNames,PS_LIST_TAIL, + psMetadataLookupStr(&status,coldef,"ENTRY"),0,"", + psMetadataLookupStr(&status,coldef,"COLNAME")); + psMetadataAddStr(colTypes,PS_LIST_TAIL, + psMetadataLookupStr(&status,coldef,"ENTRY"),0,"", + psMetadataLookupStr(&status,coldef,"DATATYPE")); + } + psFree(iter); + + if (!columnlist) { + psError(PS_ERR_UNKNOWN, false, "Failed to find a columnlist"); + psFree(whereClause); + psFree(colNames); + psFree(colTypes); + return(false); + } + + // Find the contents that comprise this run + psString contentQuery = pxDataGet("dqstatstool_get_contents.sql"); + if (!contentQuery) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retrieve sql statement (dqstatstool_get_contents.sql)"); + psFree(whereClause); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + return(false); + } + + psStringAppend(&contentQuery, " AND %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, contentQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contentQuery); + return(false); + } + psFree(contentQuery); + + psArray *contents = p_psDBFetchResult(config->dbh); + if (!contents) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + return(false); + } + if (!psArrayLength(contents)) { + psWarning("dqstatstool No rows found"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contents); + return(false); + } + + // Grab data from the database for a row of the bundle. + psString query = pxDataGet("dqstatstool_createbundle.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retrieve sql statement"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contents); + return(false); + } + // From disttool.c: + // "psStringSubstitute fails unless the input is a psString which it determines by + // comparing the memory blocks free function to an expected value. + // pxDataGet uses psSlurp which leaves a different free function on the memory block. + // To work around this make a copy of the query before doing the substitution." + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; + psStringSubstitute(&query,columnlist, "@COLLIST@"); + psFree(columnlist); + + psArray *outTable = psArrayAllocEmpty(contents->n); + for (long i = 0; i < contents->n; i++) { // Loop over content entries. + psMetadata *row = contents->data[i]; + + // Limit it just to the stuff we care about for this row. + bool status; + psString MYquery = psStringCopy(query); + psMetadata *where = psMetadataAlloc(); + + psMetadataAddS64(where,PS_LIST_TAIL, "rawExp.exp_id", 0, "==", psMetadataLookupS64(&status,row,"exp_id")); + psMetadataAddS64(where,PS_LIST_TAIL, "chipRun.chip_id", 0, "==", psMetadataLookupS64(&status,row,"chip_id")); + psMetadataAddS64(where,PS_LIST_TAIL, "camRun.cam_id", 0, "==", psMetadataLookupS64(&status,row,"cam_id")); + // This may not be real, so we need to check first. Is there a better way to check for a null here? + if (psMetadataLookupS64(&status,row,"warp_id") && (psMetadataLookupS64(&status,row,"warp_id") != 9223372036854775807)) { + psMetadataAddS64(where,PS_LIST_TAIL, "warpRun.warp_id", 0, "==", psMetadataLookupS64(&status,row,"warp_id")); + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&MYquery, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + // printf("%s\n",MYquery); + + if (!p_psDBRunQuery(config->dbh, MYquery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(MYquery); + psFree(query); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(outTable); + return(false); + } + psFree(MYquery); + + psArray *statsoutput = p_psDBFetchResult(config->dbh); + if (!statsoutput) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + return(false); + } + if (!psArrayLength(statsoutput)) { + psWarning("dqstatstool No stats rows found"); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + return(false); + } + + // Magic happens and the fits table is filled. + psMetadata *statsrow = statsoutput->data[0]; // Because there should only be one row. + + psMetadataIterator *statIter = psMetadataIteratorAlloc(statsrow,PS_LIST_HEAD,NULL); + + psMetadataItem *rowItem = NULL; + psMetadata *tableRow = psMetadataAlloc(); + // set args -dbname gpc1 -createbundle -dqstats_id 2 -uri /tmp/czwtest.fits + while ((rowItem = psMetadataGetAndIncrement(statIter))) { + bool success = false; + + psString colType = psMetadataLookupStr(&status,colTypes,rowItem->name); + if (!colType) { + psError(PS_ERR_UNKNOWN,false, "Could not find type for %s",rowItem->name); + return(false); + } + psString colName = psMetadataLookupStr(&status,colNames,rowItem->name); + if (!colName) { + psError(PS_ERR_UNKNOWN,false, "Could not find name for %s",rowItem->name); + return(false); + } + + if (!strcmp(colType,"PS_DATA_BOOL")) { + success = psMetadataAddBool(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.B); + } + else if (!strcmp(colType,"PS_DATA_S8")) { + success = psMetadataAddS8(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S8); + } + else if (!strcmp(colType,"PS_DATA_S16")) { + success = psMetadataAddS16(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S16); + } + else if (!strcmp(colType,"PS_DATA_S32")) { + success = psMetadataAddS32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S32); + } + else if (!strcmp(colType,"PS_DATA_S64")) { + success = psMetadataAddS64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S64); + } + else if (!strcmp(colType,"PS_DATA_U8")) { + success = psMetadataAddU8(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U8); + } + else if (!strcmp(colType,"PS_DATA_U16")) { + success = psMetadataAddU16(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U16); + } + else if (!strcmp(colType,"PS_DATA_U32")) { + success = psMetadataAddU32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U32); + } + else if (!strcmp(colType,"PS_DATA_U64")) { + success = psMetadataAddU64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U64); + } + else if (!strcmp(colType,"PS_DATA_F32")) { + success = psMetadataAddF32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.F32); + } + else if (!strcmp(colType,"PS_DATA_F64")) { + success = psMetadataAddF64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.F64); + } + else if (!strcmp(colType,"PS_DATA_STRING")) { + success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.str); + } + + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error constructing fits table row."); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + psFree(colType); + psFree(colName); + return(false); + } + } + + psArrayAdd(outTable,0,tableRow); + psFree(tableRow); + } + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + // Define the fits table here. + + psFits *Table = psFitsOpen(bundleUri,"w"); + if (!Table) { + psError(PS_ERR_UNKNOWN, false, "Could not open the fits table for writing."); + psFree(outTable); + return(false); + } + + psMetadata *header = psMetadataAlloc(); + psMetadataAddS64(header, PS_LIST_TAIL, "DQSTATS_ID", 0, "ID number of DQSTATS run", dqstats_id); + + if (!psFitsWriteTable(Table,header,outTable,"DQSTATS")) { + psError(PS_ERR_UNKNOWN, false, "Unable to write table to fits file."); + psFree(outTable); + psFree(Table); + psFree(header); + return(false); + } + psFree(outTable); + psFree(header); + + // Tear down, cleanup. + psFitsClose(Table); + return(true); +} + + +static bool updaterunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", "dqstatsRun.dqstats_id", "=="); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", "state", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", "fault", "=="); + + if (state && ! pxIsValidState(state)) { + psError(PXTOOLS_ERR_DATA, false, "pxIsValidState failed"); + return(false); + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + PXOPT_COPY_S64(config->args, where, "-state", "state", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return(false); + } + + psString query = psStringCopy("UPDATE dqstatsRun "); + psStringAppend(&query," SET state = '%s' ", state); + if (fault) { + psStringAppend(&query, ", fault = %d ", fault); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query," WHERE %s ",whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + psFree(query); + psFree(where); + psFree(whereClause); + + return(true); +} + +static bool revertrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return(false); + } + + psString query = psStringCopy("UPDATE dqstatsRun SET fault = 0 "); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query," WHERE fault != 0 AND %s ",whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + psFree(query); + psFree(whereClause); + psFree(where); + + return(true); +} + + + + + + + Index: branches/eam_branches/20091201/ippTools/src/dqstatstool.h =================================================================== --- branches/eam_branches/20091201/ippTools/src/dqstatstool.h (revision 26857) +++ branches/eam_branches/20091201/ippTools/src/dqstatstool.h (revision 26857) @@ -0,0 +1,18 @@ +#ifndef DQSTATS_H +#define DQSTATS_H 1 + +#include "pxtools.h" + +typedef enum { + DQSTATS_MODE_NONE = 0x0, + DQSTATS_MODE_DEFINEBYQUERY, + DQSTATS_MODE_PENDINGBUNDLE, + DQSTATS_MODE_CREATEBUNDLE, + DQSTATS_MODE_CLEANBUNDLE, + DQSTATS_MODE_UPDATERUN, + DQSTATS_MODE_REVERTRUN, +} dqstatstoolMode; + +pxConfig *dqstatstoolConfig(pxConfig *config, int argc, char **argv); + +#endif Index: branches/eam_branches/20091201/ippTools/src/dqstatstoolConfig.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/dqstatstoolConfig.c (revision 26857) +++ branches/eam_branches/20091201/ippTools/src/dqstatstoolConfig.c (revision 26857) @@ -0,0 +1,99 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include "pxtools.h" +#include "pxdqstats.h" +#include "dqstatstool.h" + +pxConfig *dqstatstoolConfig(pxConfig *config, int argc, char **argv) { + + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, "DQSTATS"); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return(NULL); + } + psTime *now = psTimeGetNow(PS_TIME_TAI); + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "set run label", NULL); + + pxdqstatsSetSearchArgs(definebyqueryArgs); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-force", 0, "invalidate old entries that match", false); + psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit bundle size to N rows", 0); + + // -createbundle + psMetadata *createbundleArgs = psMetadataAlloc(); + psMetadataAddS64(createbundleArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddStr(createbundleArgs, PS_LIST_TAIL, "-uri", 0, "output table URI", 0); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by dqstatsRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set dqstatsRun stats", NULL); + + // -pendingbundle + psMetadata *pendingbundleArgs = psMetadataAlloc(); + psMetadataAddS64(pendingbundleArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddU64(pendingbundleArgs, PS_LIST_TAIL, "-limit", 0, "limit number of bundles to N", 0); + psMetadataAddBool(pendingbundleArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(pendingbundleArgs, PS_LIST_TAIL, "-label", 0, "search by dqstatsRun.label", NULL); + + // -revertrun + psMetadata *revertrunArgs = psMetadataAlloc(); + psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label", 0, "search by dqstatsRun.label", NULL); + psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", DQSTATS_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change dqstats run properties", DQSTATS_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-createbundle", "create the bundle for a run", DQSTATS_MODE_CREATEBUNDLE, createbundleArgs); + PXOPT_ADD_MODE("-pendingbundle", "list bundles needing creation", DQSTATS_MODE_PENDINGBUNDLE, pendingbundleArgs); + PXOPT_ADD_MODE("-revertrun", "revert runs with errors", DQSTATS_MODE_REVERTRUN, revertrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return(NULL); + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return(NULL); + } + + // config = psMetadataConfigRead(config,1,"dqstatsTool.config",1); + + return config; +} + + + + Index: branches/eam_branches/20091201/ippTools/src/flatcorr.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/flatcorr.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/flatcorr.c (revision 26857) @@ -535,4 +535,5 @@ row->tess_id, "camera", + row->magicked, NULL // note does not propragate )) { Index: branches/eam_branches/20091201/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/magicdstool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/magicdstool.c (revision 26857) @@ -44,5 +44,5 @@ static bool tocleanupMode(pxConfig *config); -static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state); +static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state); # define MODECASE(caseName, func) \ @@ -326,8 +326,15 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + // optional + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + psString setString = NULL; + if (set_label) { + psStringAppend(&setString, ", label = '%s'", set_label); + } + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); if (magic_ds_id) { - return setmagicDSRunState(config, magic_ds_id, NULL, state); + return setmagicDSRunState(config, magic_ds_id, setString, NULL, state); } else if (!strcmp(state, "full")) { @@ -336,4 +343,5 @@ } // we can transition by query as well + psMetadata *where = psMetadataAlloc(); @@ -342,5 +350,4 @@ PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - if (psListLength(where->list) < 2) { psError(PS_ERR_UNKNOWN, true, "at least 2 search arguments are required"); @@ -353,5 +360,5 @@ psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0); } - bool result = setmagicDSRunState(config, magic_ds_id, where, state); + bool result = setmagicDSRunState(config, magic_ds_id, setString, where, state); psFree(where); @@ -445,5 +452,5 @@ { // first query the magicDSRun to find the stage and the stage_id - psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64; + psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { @@ -472,4 +479,5 @@ psString stage= psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); ippStage stageNum = ippStringToStage(stage); @@ -499,5 +507,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -518,5 +526,5 @@ { // first query the magicDSRun to find the stage and the stage_id - psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64; + psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { @@ -545,4 +553,5 @@ psString stage = psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); ippStage stageNum = ippStringToStage(stage); @@ -570,5 +579,5 @@ return false; } - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -598,4 +607,6 @@ PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false); PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false); + PXOPT_LOOKUP_F32(streak_frac, config->args, "-streak_frac", false, false); + PXOPT_LOOKUP_F32(run_time, config->args, "-run_time", false, false); if (setmagicked && (fault != 0)) { @@ -620,5 +631,14 @@ } - if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault, "full")) { + if (!magicDSFileInsert(config->dbh, + magic_ds_id, + component, + backup_path_base, + recovery_path_base, + streak_frac, + run_time, + fault, + "full" // data_state + )) { // rollback if (!psDBRollback(config->dbh)) { @@ -699,5 +719,5 @@ // set magicDSRun.state to 'full' - if (!setmagicDSRunState(config, magic_ds_id, NULL, "full")) { + if (!setmagicDSRunState(config, magic_ds_id, NULL, NULL, "full")) { psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64, magic_ds_id); @@ -901,4 +921,6 @@ if (!((strcmp(state, "new") == 0) || (strcmp(state, "full") == 0) || + (strcmp(state, "failed_revert") == 0) || + (strcmp(state, "failed_cleanup") == 0) || (strcmp(state, "restored") == 0) || (strcmp(state, "censored") == 0) || @@ -914,5 +936,5 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psMetadata *where, const char *state) +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psString extraSetStr, psMetadata *where, const char *state) { PS_ASSERT_PTR_NON_NULL(state, false); @@ -924,5 +946,6 @@ } - psString query = psStringCopy("UPDATE magicDSRun SET state = '%s'"); + psString query = NULL; + psStringAppend(&query, "UPDATE magicDSRun SET state = '%s' %s\n", state, extraSetStr ? extraSetStr : ""); if (magic_ds_id) { psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id); @@ -936,5 +959,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, state)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for magic_id %" PRId64, magic_ds_id); Index: branches/eam_branches/20091201/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/magicdstoolConfig.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/magicdstoolConfig.c (revision 26857) @@ -74,4 +74,5 @@ psMetadata *updaterunArgs = psMetadataAlloc(); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); @@ -102,4 +103,6 @@ psMetadataAddStr(adddestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL); psMetadataAddBool(adddestreakedfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false); + psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-streak_frac", 0, "set fraction of pixels masked by streaks", 0); + psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-run_time", 0, "set the streaksremove run time for component ", 0); psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); Index: branches/eam_branches/20091201/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/pstamptool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/pstamptool.c (revision 26857) @@ -925,4 +925,15 @@ PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + + // start a transaction eraly so it will contain any row level locks + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psString query = pxDataGet("pstamptool_pendingdependent.sql"); if (!query) { @@ -930,4 +941,14 @@ return false; } + psString whereClause = psDBGenerateWhereSQL(where, "WHERE"); + if (!no_create) { + // This will lock the row until the transaction is committed + psStringAppend(&query, " %s FOR UPDATE", whereClause); + } else { + psStringAppend(&query, " %s", whereClause); + } + psFree(whereClause); + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -947,13 +968,24 @@ if (!dep_id) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); return false; } printf("%" PRId64 "\n", dep_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return true; } if (no_create) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; } // no existing dependent that matches, insert one + // Since we have multiple processes running jobs we have a + // race condition here so that's why we need to lock the table if (!pstampDependentInsert( @@ -967,10 +999,21 @@ no_magic )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } psError(PS_ERR_UNKNOWN, false, "failed to insert pstampDependent"); return false; } + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psS64 dep_id = psDBLastInsertID(config->dbh); - printf("%" PRId64 "\n", dep_id); Index: branches/eam_branches/20091201/ippTools/src/pxcam.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/pxcam.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/pxcam.c (revision 26857) @@ -152,4 +152,5 @@ char *tess_id, char *end_stage, + psS64 magicked, char *note) { @@ -168,5 +169,5 @@ // queue the exp - // XXX chip_id is being cast here work around psS64 have a different type + // XXX chip_id and magicked are being cast here work around psS64 have a different type // different on 32/64 if (!p_psDBRunQueryF(config->dbh, query, @@ -182,4 +183,5 @@ tess_id ? tess_id : "NULL", end_stage ? end_stage : "NULL", + (long long) magicked, note ? note : "NULL", (long long)chip_id Index: branches/eam_branches/20091201/ippTools/src/pxcam.h =================================================================== --- branches/eam_branches/20091201/ippTools/src/pxcam.h (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/pxcam.h (revision 26857) @@ -40,4 +40,5 @@ char *tess_id, char *end_stage, + psS64 magicked, char *note); Index: branches/eam_branches/20091201/ippTools/src/pxchip.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/pxchip.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/pxchip.c (revision 26857) @@ -129,5 +129,5 @@ } -bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked) +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -140,5 +140,5 @@ } - char *query = "UPDATE chipRun SET state = '%s', magicked = %d WHERE chip_id = %" PRId64; + char *query = "UPDATE chipRun SET state = '%s', magicked = %" PRId64 " WHERE chip_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, state, magicked, chip_id)) { psError(PS_ERR_UNKNOWN, false, @@ -174,4 +174,9 @@ psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (!strcmp(state, "cleaned") || !strcmp(state, "purged")) { + // if magicked is non-zero set it to -1 + psStringAppend(&query, ", chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"); + } if (where && psListLength(where->list) > 0) { @@ -219,4 +224,7 @@ psStringAppend(&query, " %s", whereClause); psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; } Index: branches/eam_branches/20091201/ippTools/src/pxchip.h =================================================================== --- branches/eam_branches/20091201/ippTools/src/pxchip.h (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/pxchip.h (revision 26857) @@ -27,5 +27,5 @@ -bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked); +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked); bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); Index: branches/eam_branches/20091201/ippTools/src/pxdqstats.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/pxdqstats.c (revision 26857) +++ branches/eam_branches/20091201/ippTools/src/pxdqstats.c (revision 26857) @@ -0,0 +1,103 @@ +/* pxdqstats.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdqstats.h" + +bool pxdqstatsSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + + return true; +} + +bool pxdqstatsGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + + return true; +} Index: branches/eam_branches/20091201/ippTools/src/pxdqstats.h =================================================================== --- branches/eam_branches/20091201/ippTools/src/pxdqstats.h (revision 26857) +++ branches/eam_branches/20091201/ippTools/src/pxdqstats.h (revision 26857) @@ -0,0 +1,15 @@ +/* pxdqstats.h + */ + +#ifndef PXDQSTATS_H +#define PXDQSTATS_H 1 + +#include + +#include "pxtools.h" + +bool pxdqstatsSetSearchArgs (psMetadata *md); +bool pxdqstatsGetSearchArgs (pxConfig *config, psMetadata *where); + + +#endif // PXDQSTATS_H Index: branches/eam_branches/20091201/ippTools/src/pxmagic.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/pxmagic.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/pxmagic.c (revision 26857) @@ -85,5 +85,5 @@ } if (destreaked) { - psStringAppend(out, " AND %s.magicked != 0", table); + psStringAppend(out, " AND %s.magicked > 0", table); } return true; Index: branches/eam_branches/20091201/ippTools/src/pztool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/pztool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/pztool.c (revision 26857) @@ -117,5 +117,6 @@ telescope, uri, - NULL // epoch + NULL, // epoch + 0 // use_compress )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -783,5 +784,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); Index: branches/eam_branches/20091201/ippTools/src/receivetool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/receivetool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/receivetool.c (revision 26857) @@ -105,4 +105,5 @@ // optional PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(last, config->args, "-last", false, false); PXOPT_LOOKUP_STR(status_product, config->args, "-status_product", false, false); @@ -110,5 +111,5 @@ PXOPT_LOOKUP_STR(ds_dbhost, config->args, "-ds_dbhost", false, false); - if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, comment, last, status_product, ds_dbname, ds_dbhost)) { + if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, state, comment, last, status_product, ds_dbname, ds_dbhost)) { psError(PS_ERR_UNKNOWN, false, "Database error"); return false; @@ -142,5 +143,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } Index: branches/eam_branches/20091201/ippTools/src/receivetoolConfig.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/receivetoolConfig.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/receivetoolConfig.c (revision 26857) @@ -49,4 +49,5 @@ psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-state", 0, "define state", "enabled"); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-last", 0, "define last fileset", NULL); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-status_product", 0, "define status_product", NULL); Index: branches/eam_branches/20091201/ippTools/src/warptool.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/warptool.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/warptool.c (revision 26857) @@ -1193,7 +1193,8 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -1646,7 +1647,16 @@ } - // note only updates if warpRun.state = run_state - // XXX note that we have removed this constraint for now - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) { + char *set_magicked_skyfile = ""; + char *set_magicked_run = ""; + if (!strcmp(data_state, "full")) { + set_magicked_skyfile = "\n , warpSkyfile.magicked = IF((warpSkyfile.magicked < 0 AND chipRun.magicked = 0), warpSkyfile.magicked, chipRun.magicked)"; + set_magicked_run = "\n, warpRun.magicked = IF((warpRun.magicked < 0 AND chipRun.magicked = 0), warpRun.magicked, chipRun.magicked)"; + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + set_magicked_skyfile = "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1)"; + set_magicked_run = "\n, warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)"; + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); // rollback @@ -1660,5 +1670,5 @@ query = pxDataGet("warptool_change_run_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) { // rollback if (!psDBRollback(config->dbh)) { Index: branches/eam_branches/20091201/ippTools/src/warptoolConfig.c =================================================================== --- branches/eam_branches/20091201/ippTools/src/warptoolConfig.c (revision 26295) +++ branches/eam_branches/20091201/ippTools/src/warptoolConfig.c (revision 26857) @@ -212,5 +212,5 @@ psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); - psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked", 0, "define if this skycell has been magicked", false); + psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-magicked", 0, "define if this skycell has been magicked", 0); psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); @@ -230,4 +230,5 @@ psMetadataAddS16(warpedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); // add magic related arguments pxmagicAddArguments(warpedArgs); @@ -310,5 +311,5 @@ psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); - // -toupdateskyfile + // -updateskyfile psMetadata *updateskyfileArgs = psMetadataAlloc(); psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0);