Index: branches/tap_branches/ippTools/src/Makefile.am =================================================================== --- branches/tap_branches/ippTools/src/Makefile.am (revision 25900) +++ branches/tap_branches/ippTools/src/Makefile.am (revision 27838) @@ -9,4 +9,5 @@ disttool \ flatcorr \ + dqstatstool \ magictool \ magicdstool \ @@ -32,4 +33,5 @@ pxdata.h \ pxfake.h \ + pxdqstats.h pxmagic.h \ pxregister.h \ @@ -52,4 +54,5 @@ flatcorr.h \ faketool.h \ + dqstatstool.h \ magictool.h \ magicdstool.h \ @@ -77,4 +80,5 @@ pxfake.c \ pxfault.c \ + pxdqstats.c \ pxmagic.c \ pxregister.c \ @@ -142,4 +146,10 @@ faketool.c \ faketoolConfig.c + +dqstatstool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +dqstatstool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +dqstatstool_SOURCES = \ + dqstatstool.c \ + dqstatstoolConfig.c magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: branches/tap_branches/ippTools/src/addtool.c =================================================================== --- branches/tap_branches/ippTools/src/addtool.c (revision 25900) +++ branches/tap_branches/ippTools/src/addtool.c (revision 27838) @@ -98,11 +98,11 @@ pxcamGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","camRun.data_group", "=="); // define using camRun label PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { + if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -115,14 +115,31 @@ PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // find the cam_id of all the exposures that we want to queue up. - psString query = pxDataGet("addtool_find_cam_id.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psString bare_query = pxDataGet("addtool_find_cam_id.sql"); + if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; } + + // prevent queueing an addRun if a given exposure has already been added to + // the given dvo database + psString dvodb_string = NULL; + if (dvodb) { + // user supplied dvodb + psStringAppend(&dvodb_string, "(previous_dvodb = '%s')", dvodb); + } else { + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(camRun.dvodb IS NOT NULL AND previous_dvodb = camRun.dvodb)"); + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE @@ -131,6 +148,13 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; } psFree(where); + + if (destreaked) { + psStringAppend(&query, " AND (camRun.magicked > 0)"); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -214,12 +238,12 @@ // queue the exp if (!pxaddQueueByCamID(config, - row->cam_id, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (label ? label : row->label), - dvodb ? dvodb : row->dvodb, + row->cam_id, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : row->dvodb, note ? note : NULL, - image_only + image_only )) { if (!psDBRollback(config->dbh)) { @@ -256,8 +280,9 @@ PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-dvodb", "addRun.dvodb", "=="); if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -266,5 +291,6 @@ // pxUpdateRun gets parameters from config->args and runs the update query - bool result = pxUpdateRun(config, where, &query, "addRun", false); + bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", + "addProcessedExp", false); psFree(query); @@ -272,36 +298,4 @@ return result; - - -#ifdef notdef - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set addRun.state to state - if (!pxaddRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set addRun.label to label - if (!pxaddRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - - psFree(where); - - return true; -#endif } @@ -322,5 +316,5 @@ psString query = pxDataGet("addtool_find_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -388,5 +382,5 @@ psString query = pxDataGet("addtool_find_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -430,5 +424,5 @@ dtime_addstar, path_base, - fault + fault ); @@ -482,5 +476,5 @@ !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); return false; } @@ -488,5 +482,5 @@ psString query = pxDataGet("addtool_find_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -496,5 +490,5 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } + } // we either add AND (condition) or WHERE (condition): @@ -502,5 +496,5 @@ // list only faulted rows psStringAppend(&query, " %s", " AND addProcessedExp.fault != 0"); - } + } if (where->list && !faulted) { // don't list faulted rows @@ -510,5 +504,5 @@ // list only faulted rows psStringAppend(&query, " %s", " WHERE addProcessedExp.fault != 0"); - } + } if (!where->list && !faulted) { // don't list faulted rows @@ -571,5 +565,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -584,5 +578,5 @@ psString query = pxDataGet("addtool_reset_faulted_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -616,5 +610,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; Index: branches/tap_branches/ippTools/src/addtoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/addtoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/addtoolConfig.c (revision 27838) @@ -42,5 +42,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -52,5 +52,7 @@ pxcamSetSearchArgs(definebyqueryArgs); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by camRun data_group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-destreaked", 0, "only queue destreaked runs", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); @@ -65,5 +67,4 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); // -updaterun @@ -72,10 +73,12 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(updaterunArgs); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by addRun label", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by addRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by addRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by addRun state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by addRun reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dvodb", 0, "search by dvodb", NULL); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dvodb", 0, "set dvodb", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); Index: branches/tap_branches/ippTools/src/caltoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/caltoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/caltoolConfig.c (revision 27838) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; Index: branches/tap_branches/ippTools/src/camtool.c =================================================================== --- branches/tap_branches/ippTools/src/camtool.c (revision 25900) +++ branches/tap_branches/ippTools/src/camtool.c (revision 27838) @@ -109,11 +109,11 @@ psMetadata *where = psMetadataAlloc(); pxcamGetSearchArgs (config, where); - pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); // define using chipRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -133,5 +133,5 @@ psString query = pxDataGet("camtool_find_chip_id.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -184,8 +184,8 @@ bool status; char *end_stage = psMetadataLookupStr(&status, md, "end_stage"); - if (end_stage && strcasecmp(end_stage, "warp")) continue; + if (end_stage && strcasecmp(end_stage, "warp")) continue; char *raw_tess_id = psMetadataLookupStr(&status, md, "tess_id"); - if (raw_tess_id || tess_id) continue; + if (raw_tess_id || tess_id) continue; char *label = psMetadataLookupStr(&status, md, "label"); @@ -193,5 +193,5 @@ if (!status) { - psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); + psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); psFree(output); return false; @@ -199,11 +199,11 @@ } - // loop over our list of camRun rows + // loop over our list of chipRun rows for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; - camRunRow *row = camRunObjectFromMetadata(md); + chipRunRow *row = chipRunObjectFromMetadata(md); if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); psFree(output); return false; @@ -222,4 +222,5 @@ tess_id ? tess_id : row->tess_id, end_stage ? end_stage : row->end_stage, + row->magicked, note )) { @@ -254,4 +255,5 @@ PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group","camRun.data_group", "=="); PXOPT_COPY_STR(config->args, where, "-state", "camRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); @@ -259,5 +261,5 @@ if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -265,7 +267,7 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "camRun", true); + bool result = pxUpdateRun(config, where, &query, "camRun", "cam_id", "camProcessedExp", true); if (!result) { - psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + psError(psErrorCodeLast(), false, "pxUpdateRun failed"); } @@ -292,5 +294,5 @@ psString query = pxDataGet("camtool_find_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -357,5 +359,5 @@ psString query = pxDataGet("camtool_find_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -479,6 +481,7 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); - - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + +// Get this from the chipRun +// PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); // generate restrictions @@ -488,5 +491,5 @@ psString query = pxDataGet("camtool_find_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -523,4 +526,7 @@ return false; } + + bool status; + psS64 chip_magicked = psMetadataLookupS64(&status, output->data[0], "chip_magicked"); camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]); @@ -603,16 +609,5 @@ } - // since there is only one exp per 'new' set camRun.state = 'full' - if (!pxcamRunSetState(config, row->cam_id, "full", magicked)) { - psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); - psFree(row); - psFree(pendingRow); - return false; - } - - // NULL for end_stage means go as far as possible - // EAM : skip here if fault != 0 - // Also, we can run fake even if tess_id is not defined - if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) { + if (fault) { psFree(row); psFree(pendingRow); @@ -623,6 +618,26 @@ return true; } + // else continue on... + + // since there is only one exp per 'new' set camRun.state = 'full' + // propagate magicked state from chipRun + if (!pxcamRunSetState(config, row->cam_id, "full", chip_magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); + psFree(row); + psFree(pendingRow); + return false; + } psFree(row); - // else continue on... + + // EAM: NULL for end_stage means go as far as possible + // Also, we can run fake even if tess_id is not defined + if (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam")) { + psFree(pendingRow); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } if (!pxfakeQueueByCamID(config, @@ -666,4 +681,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); // generate restrictions @@ -672,10 +688,20 @@ PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-data_group", "camRun.data_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - if (!psListLength(where->list) && + psString where2 = NULL; + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + if (!pxmagicAddWhere(config, &where2, "chipRun")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + if (!psListLength(where->list) && !where2 && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); return false; } @@ -683,5 +709,5 @@ psString query = pxDataGet("camtool_find_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -691,5 +717,5 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } + } // we either add AND (condition) or WHERE (condition): @@ -697,5 +723,5 @@ // list only faulted rows psStringAppend(&query, " %s", " AND camProcessedExp.fault != 0"); - } + } if (where->list && !faulted) { // don't list faulted rows @@ -705,5 +731,5 @@ // list only faulted rows psStringAppend(&query, " %s", " WHERE camProcessedExp.fault != 0"); - } + } if (!where->list && !faulted) { // don't list faulted rows @@ -711,7 +737,15 @@ } psFree(where); - - // order by cam_id so that the postage stamp parser can easliy find the 'latest' astrometry - psStringAppend(&query, " ORDER BY cam_id"); + if (where2) { + psStringAppend(&query, " %s", where2); + psFree(where2); + } + + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, " ORDER BY exp_id, cam_id DESC"); + } // treat limit == 0 as "no limit" @@ -766,5 +800,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -779,5 +813,5 @@ psString query = pxDataGet("camtool_reset_faulted_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -811,5 +845,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -951,5 +985,5 @@ psString query = pxDataGet("camtool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1016,5 +1050,5 @@ psString query = pxDataGet("camtool_pendingcleanupexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1077,5 +1111,5 @@ psString query = pxDataGet("camtool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1163,5 +1197,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/camtoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/camtoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/camtoolConfig.c (revision 27838) @@ -42,5 +42,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -73,6 +73,7 @@ pxcamSetSearchArgs(updaterunArgs); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by camRun data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -178,5 +179,10 @@ psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by camRun data_group (LIKE comparison)", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by camRun reduction class", NULL); + pxspaceAddArguments(processedexpArgs); + pxmagicAddArguments(processedexpArgs); + + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order the results for the postage stamp server", false); psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/tap_branches/ippTools/src/chiptool.c =================================================================== --- branches/tap_branches/ippTools/src/chiptool.c (revision 25900) +++ branches/tap_branches/ippTools/src/chiptool.c (revision 27838) @@ -36,4 +36,5 @@ static bool definebyqueryMode(pxConfig *config); +static bool definecopyMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool pendingimfileMode(pxConfig *config); @@ -58,4 +59,6 @@ static bool importrunMode(pxConfig *config); static bool runstateMode(pxConfig *config); +static bool setimfiletoupdateMode(pxConfig *config); +static bool listrunMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -77,4 +80,5 @@ switch (config->mode) { MODECASE(CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(CHIPTOOL_MODE_DEFINECOPY, definecopyMode); MODECASE(CHIPTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileMode); @@ -95,8 +99,10 @@ MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); - MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); + MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); MODECASE(CHIPTOOL_MODE_RUNSTATE, runstateMode); + MODECASE(CHIPTOOL_MODE_SETIMFILETOUPDATE, setimfiletoupdateMode); + MODECASE(CHIPTOOL_MODE_LISTRUN, listrunMode); default: psAbort("invalid option (this should not happen)"); @@ -120,4 +126,116 @@ } +// Queue exposures for chip processing +static bool queue_exposures(pxConfig *config, // Configuration + const psArray *exps, // Exposures information + const char *workdir, // Working directory, or NULL to inherit + const char *label, // Label, or NULL to inherit + const char *data_group, // Data group, or NULL to inherit + const char *dist_group, // Distribution group, or NULL to inherit + const char *reduction, // Reduction class, or NULL to inherit + const char *expgroup, // Exposure group + const char *dvodb, // DVO database, or NULL to inherit + const char *tess_id, // Tessellation identifier, or NULL to inherit + const char *end_stage, // End stage, or NULL to inherit + const char *note // Note + ) +{ + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // if end_stage is warp (or NULL), check for valid tess_id + for (long i = 0; i < psArrayLength(exps); i++) { + psMetadata *md = exps->data[i]; + + bool status; + char *end_stage = psMetadataLookupStr(&status, md, "end_stage"); + if (end_stage && strcasecmp(end_stage, "warp")) continue; + + char *raw_tess_id = psMetadataLookupStr(&status, md, "tess_id"); + if (raw_tess_id || tess_id) continue; + + char *label = psMetadataLookupStr(&status, md, "label"); + psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + + if (!status) { + psError(PS_ERR_UNKNOWN, false, + "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, + label, exp_id); + return false; + } + } + + +# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME) \ + PTYPE VALUE; \ + { \ + bool status; \ + VALUE = psMetadataLookup##CTYPE(&status, md, NAME); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \ + return false; \ + } \ + } + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(exps); i++) { + psMetadata *md = exps->data[i]; + + rawExpRow *row = rawExpObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + return false; + } + + GET_VALUE (psS64, S64, exp_id, "exp_id"); + GET_VALUE (psString, Str, raw_workdir, "workdir"); + GET_VALUE (psString, Str, raw_label, "label"); + GET_VALUE (psString, Str, raw_reduction, "reduction"); + // GET_VALUE (psString, Str, raw_expgroup, "expgroup"); + GET_VALUE (psString, Str, raw_dvodb, "dvodb"); + GET_VALUE (psString, Str, raw_tess_id, "tess_id"); + GET_VALUE (psString, Str, raw_end_stage, "end_stage"); + + if (!row->exp_id) { + psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id"); + return false; + } + + // queue the exp + if (!pxchipQueueByExpTag(config, + exp_id, + workdir ? workdir : raw_workdir, + label ? label : raw_label, + data_group ? data_group : (label ? label : raw_label), + dist_group, + reduction ? reduction : raw_reduction, + // expgroup ? expgroup : raw_expgroup, + // XXX how does expgroup get defined? + expgroup, + dvodb ? dvodb : raw_dvodb, + tess_id ? tess_id : raw_tess_id, + end_stage ? end_stage : raw_end_stage, + note + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} static bool definebyqueryMode(pxConfig *config) @@ -129,5 +247,5 @@ psMetadata *where = psMetadataAlloc(); pxchipGetSearchArgs (config, where); // rawExp only - pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); // define using newExp label // psListLength(where->list) is at least 1 because exp_type defaults to "object" @@ -135,5 +253,5 @@ if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -157,5 +275,5 @@ psString query = pxDataGet("chiptool_find_rawexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -197,101 +315,98 @@ } - // start a transaction so we don't end up with an exp without any associted - // imfiles - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - return false; - } - - // if end_stage is warp (or NULL), check for valid tess_id - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - bool status; - char *end_stage = psMetadataLookupStr(&status, md, "end_stage"); - if (end_stage && strcasecmp(end_stage, "warp")) continue; - - char *raw_tess_id = psMetadataLookupStr(&status, md, "tess_id"); - if (raw_tess_id || tess_id) continue; - - char *label = psMetadataLookupStr(&status, md, "label"); - psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); - - if (!status) { - psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); + if (!queue_exposures(config, output, workdir, label, data_group, dist_group, reduction, expgroup, + dvodb, tess_id, end_stage, note)) { + psError(psErrorCodeLast(), false, "Unable to queue exposures for chip."); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool definecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); + + // psListLength(where->list) is at least 1 because exp_type defaults to "object" + // so we require a list longer than 1 entry + if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + // default + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("chiptool_definecopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); return false; } - } - - -# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME) \ - PTYPE VALUE; \ - { bool status; \ - VALUE = psMetadataLookup##CTYPE(&status, md, NAME); \ - if (!status) { \ - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \ - psFree(output); \ - return false; \ - } } - - // loop over our list of exp_ids - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - rawExpRow *row = rawExpObjectFromMetadata(md); - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); - psFree(output); - return false; - } - - GET_VALUE (psS64, S64, exp_id, "exp_id"); - GET_VALUE (psString, Str, raw_workdir, "workdir"); - GET_VALUE (psString, Str, raw_label, "label"); - GET_VALUE (psString, Str, raw_reduction, "reduction"); - // GET_VALUE (psString, Str, raw_expgroup, "expgroup"); - GET_VALUE (psString, Str, raw_dvodb, "dvodb"); - GET_VALUE (psString, Str, raw_tess_id, "tess_id"); - GET_VALUE (psString, Str, raw_end_stage, "end_stage"); - - if (!row->exp_id) { - psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id"); - psFree(output); - return false; - } - - // queue the exp - if (!pxchipQueueByExpTag(config, - exp_id, - workdir ? workdir : raw_workdir, - label ? label : raw_label, - data_group ? data_group : (label ? label : raw_label), - dist_group, - reduction ? reduction : raw_reduction, - // expgroup ? expgroup : raw_expgroup, - // XXX how does expgroup get defined? - expgroup, - dvodb ? dvodb : raw_dvodb, - tess_id ? tess_id : raw_tess_id, - end_stage ? end_stage : raw_end_stage, - note - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue exp_id: %" PRId64, exp_id); - psFree(output); - return false; - } + psFree(output); + return true; + } + + if (!queue_exposures(config, output, workdir, label, data_group, dist_group, reduction, expgroup, + dvodb, tess_id, end_stage, note)) { + psError(psErrorCodeLast(), false, "Unable to queue exposures for chip."); + psFree(output); + return false; } psFree(output); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } return true; @@ -314,13 +429,13 @@ if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id)"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) -- join hook %s\n"); // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "chipRun", true); + bool result = pxUpdateRun(config, where, &query, "chipRun", "chip_id", "chipProcessedImfile", true); if (!result) { - psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + psError(psErrorCodeLast(), false, "pxUpdateRun failed"); } @@ -346,5 +461,5 @@ psString query = pxDataGet("chiptool_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -586,4 +701,8 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(allfiles, config->args, "-allfiles", false); + if (allfiles) { + faulted = false; + } psMetadata *where = psMetadataAlloc(); @@ -594,4 +713,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "=="); @@ -600,5 +720,5 @@ // add cuts on ra and decl if supplied if (!pxspaceAddWhere(config, &where2, "rawExp")) { - psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); return false; } @@ -606,5 +726,5 @@ psString query = pxDataGet("chiptool_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -612,5 +732,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); psFree(whereClause); } else if (psMetadataLookupBool(NULL, config->args, "-all") || (faulted || where2)) { @@ -618,5 +738,5 @@ } else { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); return false; } @@ -626,5 +746,5 @@ // list only faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); - } else { + } else if (!allfiles) { // don't list faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); @@ -683,5 +803,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -689,5 +809,11 @@ psString query = pxDataGet("chiptool_revertprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + psString query_update = pxDataGet("chiptool_revertupdatedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -697,4 +823,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_update, " AND %s", whereClause); psFree(whereClause); } @@ -708,4 +835,10 @@ } psFree(query); + if (!p_psDBRunQuery(config->dbh, query_update)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_update); + return false; + } + psFree(query_update); return true; @@ -720,8 +853,12 @@ PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); - + if (psListLength(where->list) == 0) { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + if (!state) { PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); @@ -732,10 +869,10 @@ else { if (!pxchipProcessedImfileSetStateByQuery(config,where,state)) { - psError(PS_ERR_UNKNOWN, false, "failed to set chipProcessedImfile state"); - return(false); + psError(PS_ERR_UNKNOWN, false, "failed to set chipProcessedImfile state"); + return(false); } } - + return true; } @@ -839,5 +976,5 @@ psString query = pxDataGet("chiptool_unmasked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -917,5 +1054,5 @@ psString query = pxDataGet("chiptool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -981,5 +1118,5 @@ psString query = pxDataGet("chiptool_pendingcleanupimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1042,5 +1179,5 @@ psString query = pxDataGet("chiptool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1100,5 +1237,5 @@ // make sure that the state string is valid if (!pxIsValidState(state)) { - psError(PXTOOLS_ERR_DATA, false, "%s is not a valid state", state); + psError(PXTOOLS_ERR_CONFIG, false, "%s is not a valid state", state); return false; } @@ -1113,5 +1250,5 @@ psString query = pxDataGet("chiptool_run.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1175,5 +1312,5 @@ psString query = pxDataGet("chiptool_completely_processed_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1254,4 +1391,5 @@ chipRun->tess_id, chipRun->end_stage, + chipRun->magicked, NULL // note does not propagate )) { @@ -1295,13 +1433,24 @@ } - psString set_magic = ""; + psString set_magicked_imfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); if (!strcmp(data_state, "full")) { - // copy the magicked state from the input to the output when transitioning to full state - set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked"; - } - - // note only updates if chipRun.state = run_state - // XXX note that we have removed this constraint for now - if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) { + // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will + // block warp processing until destreaking has been done + // otherwise copy magicked from the rawImfile + // Same thing for chipRun/rawExp + psStringAppend(&set_magicked_imfile, "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0" + " AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)"); + psStringAppend(&set_magicked_run, "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), " + " chipRun.magicked, rawExp.magicked)"); + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + // if magicked is non-zero set it to -1 + // Once one imfile has been cleaned, the chipRun is no longer 'magicked' + psStringAppend(&set_magicked_imfile, "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)," + " chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) { psFree(query); psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1314,4 +1463,6 @@ } psFree(query); + psFree(set_magicked_imfile); + psFree(set_magicked_run); if (psDBAffectedRows(config->dbh) < 1) { psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); @@ -1320,5 +1471,5 @@ query = pxDataGet("chiptool_change_exp_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) { psFree(query); // rollback @@ -1395,5 +1546,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1431,5 +1582,5 @@ if (clean) { - bool success = true; + bool success = true; if (!strcmp(tables[i].tableName, "chipRun")) { success = pxSetStateCleaned("chipRun", "state", output); @@ -1535,5 +1686,5 @@ psString query = pxDataGet("chiptool_runstate.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1544,5 +1695,5 @@ psFree(whereClause); } else { - psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); return false; } @@ -1596,2 +1747,125 @@ return true; } + +// a very specfic function to queue a cleaned chipProcessedImfile to be updated +static bool setimfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("chiptool_setimfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , chipRun.label = '%s'", label); + } + + if (class_id) { + psStringAppend(&query, " AND (chipProcessedImfile.class_id = '%s')", class_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "chipRun.dist_group", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipRun"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + psString query = pxDataGet("chiptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); + psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || where2) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, chip_id DESC"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: branches/tap_branches/ippTools/src/chiptool.h =================================================================== --- branches/tap_branches/ippTools/src/chiptool.h (revision 25900) +++ branches/tap_branches/ippTools/src/chiptool.h (revision 27838) @@ -26,4 +26,5 @@ CHIPTOOL_MODE_NONE = 0x0, CHIPTOOL_MODE_DEFINEBYQUERY, + CHIPTOOL_MODE_DEFINECOPY, CHIPTOOL_MODE_UPDATERUN, CHIPTOOL_MODE_PENDINGIMFILE, @@ -32,4 +33,5 @@ CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, + CHIPTOOL_MODE_SETIMFILETOUPDATE, CHIPTOOL_MODE_ADVANCEEXP, CHIPTOOL_MODE_BLOCK, @@ -48,5 +50,6 @@ CHIPTOOL_MODE_EXPORTRUN, CHIPTOOL_MODE_IMPORTRUN, - CHIPTOOL_MODE_RUNSTATE + CHIPTOOL_MODE_RUNSTATE, + CHIPTOOL_MODE_LISTRUN } chiptoolMode; Index: branches/tap_branches/ippTools/src/chiptoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/chiptoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/chiptoolConfig.c (revision 27838) @@ -39,5 +39,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -63,4 +63,24 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definecopy + psMetadata *definecopyArgs = psMetadataAlloc(); + pxchipSetSearchArgs (definecopyArgs); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by rawExp label (LIKE comparison)", NULL); + + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); + + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define distribution group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updaterun @@ -77,4 +97,10 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "set dist group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "set note", NULL); + + // -setimfiletoupdate + psMetadata *setimfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setimfiletoupdateArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id (required)", NULL); + psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0,"new value for label", NULL); // -pendingimfile @@ -154,5 +180,5 @@ psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); - psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", false); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", 0); // -processedimfile @@ -164,4 +190,5 @@ psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL); pxmagicAddArguments(processedimfileArgs); @@ -169,4 +196,5 @@ psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-allfiles", 0, "only all matching imfiles regardless of fault status", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -189,4 +217,22 @@ psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + + // -processedExp + psMetadata *listrunArgs = psMetadataAlloc(); + pxchipSetSearchArgs(listrunArgs); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by chipRun state", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by chipRun dist_group (LIKE comparison)", NULL); + pxmagicAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + pxspaceAddArguments(listrunArgs); + // -block psMetadata *blockArgs = psMetadataAlloc(); @@ -292,5 +338,7 @@ PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definecopy", "create copy runs", CHIPTOOL_MODE_DEFINECOPY, definecopyArgs); PXOPT_ADD_MODE("-updaterun", "change chip run properties", CHIPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-setimfiletoupdate", "set imfile state to update", CHIPTOOL_MODE_SETIMFILETOUPDATE, setimfiletoupdateArgs); PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); PXOPT_ADD_MODE("-addprocessedimfile", "add a processed imfile", CHIPTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); @@ -299,4 +347,5 @@ PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); PXOPT_ADD_MODE("-runstate", "list the states of chip run", CHIPTOOL_MODE_RUNSTATE, runstateArgs); + PXOPT_ADD_MODE("-listrun", "list chipRuns", CHIPTOOL_MODE_LISTRUN, listrunArgs); PXOPT_ADD_MODE("-advanceexp", "advance completed exposures", CHIPTOOL_MODE_ADVANCEEXP, advanceexpArgs); PXOPT_ADD_MODE("-block", "set a label block", CHIPTOOL_MODE_BLOCK, blockArgs); @@ -317,5 +366,5 @@ if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { - psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); psFree(argSets); psFree(modes); Index: branches/tap_branches/ippTools/src/detselect.c =================================================================== --- branches/tap_branches/ippTools/src/detselect.c (revision 25900) +++ branches/tap_branches/ippTools/src/detselect.c (revision 27838) @@ -135,69 +135,5 @@ psString query = pxDataGet("detselect_search.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // we choose the single detrend image which matches all criteria and has - // the latest insertion date - - // unless explicitly specified by the user, list all possible matches - if (!unlimit) { - psStringAppend(&query, " ORDER BY registered DESC LIMIT 1"); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - // XXX check psError here - psError(PS_ERR_UNKNOWN, false, "no detrend exposures found"); - psFree(output); - return true; - } - - // negate simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "detExp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - -static bool selectMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); - PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - - psString query = pxDataGet("detselect_select.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -211,4 +147,68 @@ psFree(where); + // we choose the single detrend image which matches all criteria and has + // the latest insertion date + + // unless explicitly specified by the user, list all possible matches + if (!unlimit) { + psStringAppend(&query, " ORDER BY registered DESC LIMIT 1"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detrend exposures found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool selectMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("detselect_select.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/tap_branches/ippTools/src/dettool.c =================================================================== --- branches/tap_branches/ippTools/src/dettool.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool.c (revision 27838) @@ -94,4 +94,5 @@ MODECASE(DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, pendingcleanup_processedexpMode); MODECASE(DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, donecleanup_processedexpMode); + MODECASE(DETTOOL_MODE_UPDATESTATE_PROCESSED, updatestateprocessedMode); // stacked MODECASE(DETTOOL_MODE_TOSTACKED, tostackedMode); @@ -142,4 +143,5 @@ MODECASE(DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, pendingcleanup_residexpMode); MODECASE(DETTOOL_MODE_DONECLEANUP_RESIDEXP, donecleanup_residexpMode); + MODECASE(DETTOOL_MODE_UPDATESTATE_RESID, updatestateresidMode); // detrunsummary MODECASE(DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryMode); @@ -150,5 +152,5 @@ MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); MODECASE(DETTOOL_MODE_RERUN, rerunMode); - MODECASE(DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryMode); // register MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); @@ -185,8 +187,8 @@ psString query = pxDataGet("dettool_pending.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); @@ -258,10 +260,10 @@ psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); return false; - } + } if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); return false; - } - + } + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); @@ -308,5 +310,5 @@ psMetadataItem *mItem = NULL; while ((mItem = psListGetAndIncrement(iter))) { - psS64 exp_id = mItem->data.S64; + psS64 exp_id = mItem->data.S64; if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); @@ -317,13 +319,13 @@ } psFree(iter); - } + } if (item->type == PS_DATA_S64) { - psS64 exp_id = item->data.S64; - if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - } + psS64 exp_id = item->data.S64; + if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } if (psListLength(where->list) < 1) { @@ -505,9 +507,9 @@ psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); return false; - } + } if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); return false; - } + } PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); @@ -550,4 +552,5 @@ PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "airmass", "<="); PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_max", "sat_pixel_frac", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_min", "sat_pixel_frac", ">="); PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "exp_time", ">="); PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "exp_time", "<="); @@ -673,5 +676,5 @@ label, ref_det_id, - ref_iter + ref_iter ); psS64 det_id = psDBLastInsertID(config->dbh); @@ -742,5 +745,5 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - + // XXX this mode is not well-tested: probably need to specify iteration here PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required @@ -759,9 +762,9 @@ psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); return false; - } + } if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); return false; - } + } // the new detRun may have different values for these limits: @@ -817,5 +820,5 @@ if (iteration < 0) { - iteration = detRun->iteration; + iteration = detRun->iteration; } @@ -977,5 +980,5 @@ psString query = pxDataGet("dettool_definebydetrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -989,5 +992,5 @@ if (only_accepted) { - psStringAppend(&query, " AND accept = 1"); + psStringAppend(&query, " AND accept = 1"); } @@ -1068,5 +1071,5 @@ psString query = pxDataGet("dettool_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1138,5 +1141,5 @@ psString query = pxDataGet("dettool_childlessrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -1212,5 +1215,5 @@ psString query = pxDataGet("dettool_input.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -1264,5 +1267,5 @@ psString query = pxDataGet("dettool_raw.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1338,5 +1341,5 @@ } -// used by updatedetrunMode +// used by updatedetrunMode bool startNewIteration(pxConfig *config, psS64 det_id) { @@ -1345,5 +1348,5 @@ psString query = pxDataGet("dettool_start_new_iteration.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1564,5 +1567,5 @@ } detInputExpRow *newInputExp = detInputExpRowAlloc( - det_id, + det_id, newIteration, inputExp->exp_id, @@ -1662,6 +1665,6 @@ solang_max, label, - ref_det_id, - ref_iter + ref_det_id, + ref_iter )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1713,5 +1716,5 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required @@ -1728,20 +1731,20 @@ if (!detRegisteredImfileInsert(config->dbh, - det_id, - 0, // the iteration is fixed at 0 - class_id, - uri, - bg, - bg_stdev, - bg_mean_stdev, - user_1, - user_2, - user_3, - user_4, - user_5, - path_base, - "full", - 0 // fault code - )) { + det_id, + 0, // the iteration is fixed at 0 + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + 0 // fault code + )) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -1858,47 +1861,113 @@ -bool setProcessedImfileDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *class_id, const char *data_state) +bool setProcessedImfileDataState(pxConfig *config, psMetadata *where, const char *data_state) { PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where, false); PS_ASSERT_PTR_NON_NULL(data_state, false); - PS_ASSERT_PTR_NON_NULL(class_id, false); if (!isValidDataState (data_state)) return false; - - char *query = "UPDATE detProcessedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND exp_id = %" PRId64 - " AND class_id = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, exp_id, class_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", exp_id %" PRId64 ", class_id %s", - det_id, exp_id, class_id); - return false; - } - - return true; -} - -bool setProcessedExpDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *data_state) + + psString query = psStringCopy("UPDATE detProcessedImfile SET data_state = '%s'"); + if (where && psListLength(where->list) > 0) { + psString whereClause = psDBGenerateWhereSQL(where,NULL); + psStringAppend(&query," %s",whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + return(true); +} + +bool setProcessedExpDataState(pxConfig *config, psMetadata *where, const char *data_state) { PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where, false); PS_ASSERT_PTR_NON_NULL(data_state, false); if (!isValidDataState (data_state)) return false; - char *query = "UPDATE detProcessedExp SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND exp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, exp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", exp_id %" PRId64, - det_id, exp_id); - return false; - } - - return true; -} - - + psString query = psStringCopy("UPDATE detProcessedExp SET data_state = '%s'"); + if (where && psListLength(where->list) > 0) { + psString whereClause = psDBGenerateWhereSQL(where,NULL); + psStringAppend(&query," %s",whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + return(true); +} + +bool setResidImfileDataState(pxConfig *config, psMetadata *where, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where,false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + psString query = psStringCopy("UPDATE detResidImfile SET data_state = '%s'"); + if (where && psListLength(where->list) > 0) { + psString whereClause = psDBGenerateWhereSQL(where,NULL); + psStringAppend(&query," %s",whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + return(true); +} +// This function apparently not used anymore. +bool setResidExpDataState(pxConfig *config, psMetadata *where, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where,false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + psString query = psStringCopy("UPDATE detResidExp SET data_state = '%s'"); + if (where && psListLength(where->list) > 0) { + psString whereClause = psDBGenerateWhereSQL(where,NULL); + psStringAppend(&query," %s",whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + return(true); +} + +// Not yet updated for cleaning bool setStackedImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state) { @@ -1910,11 +1979,11 @@ char *query = "UPDATE detStackedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND class_id = '%s'"; + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32 "class_id %s", - det_id, iteration, class_id); + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 "class_id %s", + det_id, iteration, class_id); return false; } @@ -1932,12 +2001,12 @@ char *query = "UPDATE detNormalizedStatImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND class_id = '%s'"; + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = '%s'"; /* fprintf(stderr,"DETTOOL SAYS: %s\n",query); */ if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration,class_id)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32, - det_id, iteration); + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, + det_id, iteration); return false; } @@ -1955,11 +2024,11 @@ char *query = "UPDATE detNormalizedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND class_id = '%s'"; + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32 " class %s", - det_id, iteration, class_id); + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 " class %s", + det_id, iteration, class_id); return false; } @@ -1976,10 +2045,10 @@ char *query = "UPDATE detNormalizedExp SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32, - det_id, iteration); + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, + det_id, iteration); return false; } @@ -1987,48 +2056,5 @@ return true; } - -bool setResidImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *class_id, const char *data_state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(data_state, false); - PS_ASSERT_PTR_NON_NULL(class_id, false); - - if (!isValidDataState (data_state)) return false; - - char *query = "UPDATE detResidImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND exp_id = %" PRId64 - " AND class_id = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, exp_id, class_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32 ", exp_id %" PRId64 ", class_id %s", - det_id, iteration, exp_id, class_id); - return false; - } - - return true; -} - -bool setResidExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *data_state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(data_state, false); - - if (!isValidDataState (data_state)) return false; - - char *query = "UPDATE detResidExp SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND exp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, exp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32 ", exp_id %" PRId64, - det_id, iteration, exp_id); - return false; - } - - return true; -} +// End not yet modified for cleanup. bool exportrunMode(pxConfig *config) @@ -2038,5 +2064,5 @@ char sqlFilename[80]; } ExportTable; - + int numExportTables = 12; @@ -2074,5 +2100,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -2116,5 +2142,5 @@ } psFree(output); - + } @@ -2127,7 +2153,7 @@ { unsigned int nFail; - + int numImportTables = 11; - + char tables[11] [80] = {"detInputExp", "detNormalizedExp", "detNormalizedImfile", "detNormalizedStatImfile", "detProcessedExp", @@ -2136,5 +2162,5 @@ PS_ASSERT_PTR_NON_NULL(config, NULL); - + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); @@ -2143,5 +2169,5 @@ fprintf (stdout, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - + psMetadataItem *item = psMetadataLookup (input, "detRun"); psAssert (item, "entry not in input?"); @@ -2153,5 +2179,5 @@ detRunRow *detRun = detRunObjectFromMetadata (entry->data.md); detRunInsertObject (config->dbh, detRun); - + // fprintf (stdout, "---- det run ----\n"); // psMetadataPrint (stderr, entry->data.md, 1); @@ -2161,5 +2187,5 @@ psAssert (item, "entry not in input?"); psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - + switch (i) { case 0: @@ -2175,5 +2201,5 @@ } break; - + case 1: for (int i = 0; i < item->data.list->n; i++) { @@ -2188,5 +2214,5 @@ } break; - + case 2: for (int i = 0; i < item->data.list->n; i++) { @@ -2201,5 +2227,5 @@ } break; - + case 3: for (int i = 0; i < item->data.list->n; i++) { @@ -2214,5 +2240,5 @@ } break; - + case 4: for (int i = 0; i < item->data.list->n; i++) { @@ -2227,5 +2253,5 @@ } break; - + case 5: for (int i = 0; i < item->data.list->n; i++) { @@ -2240,5 +2266,5 @@ } break; - + case 6: for (int i = 0; i < item->data.list->n; i++) { @@ -2253,5 +2279,5 @@ } break; - + case 7: for (int i = 0; i < item->data.list->n; i++) { @@ -2266,5 +2292,5 @@ } break; - + case 8: for (int i = 0; i < item->data.list->n; i++) { @@ -2279,5 +2305,5 @@ } break; - + case 9: for (int i = 0; i < item->data.list->n; i++) { @@ -2292,5 +2318,5 @@ } break; - + case 10: for (int i = 0; i < item->data.list->n; i++) { Index: branches/tap_branches/ippTools/src/dettool.h =================================================================== --- branches/tap_branches/ippTools/src/dettool.h (revision 25900) +++ branches/tap_branches/ippTools/src/dettool.h (revision 27838) @@ -57,4 +57,5 @@ DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, + DETTOOL_MODE_UPDATESTATE_PROCESSED, DETTOOL_MODE_TOSTACKED, @@ -105,4 +106,5 @@ DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, DETTOOL_MODE_DONECLEANUP_RESIDEXP, + DETTOOL_MODE_UPDATESTATE_RESID, DETTOOL_MODE_TODETRUNSUMMARY, @@ -149,4 +151,5 @@ bool pendingcleanup_processedexpMode(pxConfig *config); bool donecleanup_processedexpMode(pxConfig *config); +bool updatestateprocessedMode(pxConfig *config); // stackedimfile @@ -203,4 +206,5 @@ bool pendingcleanup_residexpMode(pxConfig *config); bool donecleanup_residexpMode(pxConfig *config); +bool updatestateresidMode(pxConfig *config); // detrunsummary @@ -220,12 +224,13 @@ // functions to set the 'data_state' for stages -bool setProcessedImfileDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *class_id, const char *data_state); -bool setProcessedExpDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *data_state); +bool setProcessedImfileDataState(pxConfig *config, psMetadata *where, const char *data_state); +bool setProcessedExpDataState(pxConfig *config, psMetadata *where, const char *data_state); +bool setResidImfileDataState(pxConfig *config, psMetadata *where, const char *data_state); +bool setResidExpDataState(pxConfig *config, psMetadata *where, const char *data_state); + bool setStackedImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); bool setNormStatImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); bool setNormImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); bool setNormExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *data_state); -bool setResidImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *class_id, const char *data_state); -bool setResidExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *data_state); #endif // DETTOOL_H Index: branches/tap_branches/ippTools/src/dettoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/dettoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettoolConfig.c (revision 27838) @@ -31,5 +31,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -117,4 +117,5 @@ psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, "define max airmass", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_min", 0, "define min fraction of saturated pixels", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, "define min exposure time", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, "define max exposure time", NAN); @@ -305,5 +306,5 @@ // -updateprocessedimfile psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -314,5 +315,5 @@ psMetadataAddBool(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -322,5 +323,5 @@ psMetadataAddBool(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -366,5 +367,5 @@ // -updateprocessedexp psMetadata *updateprocessedexpArgs = psMetadataAlloc(); - psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); @@ -374,5 +375,5 @@ psMetadataAddBool(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); @@ -381,6 +382,12 @@ psMetadataAddBool(donecleanup_processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + + // -updatestate_processed + psMetadata *updatestateprocessedArgs = psMetadataAlloc(); + psMetadataAddS64(updatestateprocessedArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); + psMetadataAddS64(updatestateprocessedArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updatestateprocessedArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -tostacked @@ -425,5 +432,5 @@ // -updatestacked psMetadata *updatestackedArgs = psMetadataAlloc(); - psMetadataAddS64(updatestackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updatestackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -434,5 +441,5 @@ psMetadataAddBool(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -442,5 +449,5 @@ psMetadataAddBool(donecleanup_stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(donecleanup_stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -477,5 +484,5 @@ // -updatenormalizedstat psMetadata *updatenormalizedstatArgs = psMetadataAlloc(); - psMetadataAddS64(updatenormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updatenormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -486,5 +493,5 @@ psMetadataAddBool(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -494,5 +501,5 @@ psMetadataAddBool(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -539,5 +546,5 @@ // -updatenormalizedimfile psMetadata *updatenormalizedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(updatenormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updatenormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -548,5 +555,5 @@ psMetadataAddBool(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -556,5 +563,5 @@ psMetadataAddBool(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -598,5 +605,5 @@ // -updatenormalizedexp psMetadata *updatenormalizedexpArgs = psMetadataAlloc(); - psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); @@ -606,5 +613,5 @@ psMetadataAddBool(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); @@ -613,5 +620,5 @@ psMetadataAddBool(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); @@ -676,5 +683,5 @@ // -updateresidimfile psMetadata *updateresidimfileArgs = psMetadataAlloc(); - psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updateresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); @@ -686,5 +693,5 @@ psMetadataAddBool(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); @@ -695,5 +702,5 @@ psMetadataAddBool(donecleanup_residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); @@ -778,4 +785,10 @@ psMetadataAddS32(donecleanup_residexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(donecleanup_residexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + + // -updatestate_resid + psMetadata *updatestateresidArgs = psMetadataAlloc(); + psMetadataAddS64(updatestateresidArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); + psMetadataAddS64(updatestateresidArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updatestateresidArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -todetrunsummary @@ -924,4 +937,5 @@ PXOPT_ADD_MODE("-pendingcleanup_processedexp", "", DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, pendingcleanup_processedexpArgs); PXOPT_ADD_MODE("-donecleanup_processedexp", "", DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, donecleanup_processedexpArgs); + PXOPT_ADD_MODE("-updatestate_processed", "", DETTOOL_MODE_UPDATESTATE_PROCESSED, updatestateprocessedArgs); PXOPT_ADD_MODE("-tostacked", "", DETTOOL_MODE_TOSTACKED, tostackedArgs); @@ -972,4 +986,5 @@ PXOPT_ADD_MODE("-pendingcleanup_residexp", "", DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, pendingcleanup_residexpArgs); PXOPT_ADD_MODE("-donecleanup_residexp", "", DETTOOL_MODE_DONECLEANUP_RESIDEXP, donecleanup_residexpArgs); + PXOPT_ADD_MODE("-updatestate_resid", "", DETTOOL_MODE_UPDATESTATE_RESID, updatestateresidArgs); PXOPT_ADD_MODE("-todetrunsummary", "", DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryArgs); Index: branches/tap_branches/ippTools/src/dettool_correction.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_correction.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_correction.c (revision 27838) @@ -34,5 +34,5 @@ PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - // build the needed where + // build the needed where psMetadata *where = psMetadataAlloc(); psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", ref_det_id); @@ -88,6 +88,6 @@ detRun->solang_max, use_label, - detRun->det_id, // ref_det_id - detRun->iteration // ref_iter + detRun->det_id, // ref_det_id + detRun->iteration // ref_iter ); psFree(runs); @@ -140,11 +140,11 @@ psString query = pxDataGet("dettool_tocorrectimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } if (psListLength(where->list)) { - // NOTE the SQL uses an intermediate table 'det1' for this query - psString whereClause = psDBGenerateWhereConditionSQL(where, "det1"); + // NOTE the SQL uses an intermediate table 'det1' for this query + psString whereClause = psDBGenerateWhereConditionSQL(where, "det1"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -198,5 +198,5 @@ psString query = pxDataGet("dettool_tocorrectexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -248,9 +248,9 @@ return false; } - + // automatically stop completed 'correct' detRuns psString query = pxDataGet("dettool_stop_completed_correct_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/dettool_detrunsummary.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_detrunsummary.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_detrunsummary.c (revision 27838) @@ -35,5 +35,5 @@ psString query = pxDataGet("dettool_todetrunsummary.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -135,5 +135,5 @@ psString query = pxDataGet("dettool_find_completed_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -241,5 +241,5 @@ psString query = pxDataGet("dettool_detrunsummary.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -310,5 +310,5 @@ psString query = pxDataGet("dettool_revertdetrunsummary.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -348,8 +348,8 @@ psString query = pxDataGet("dettool_pendingcleanup_detrunsummary.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return(false); } - + if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); @@ -429,93 +429,93 @@ /* if (!isValidDataState(data_state)) return false; */ - - - + + + char *query_detRunSummary = "UPDATE detRunSummary SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; if (!p_psDBRunQueryF(config->dbh, query_detRunSummary, data_state, det_id,iteration)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return false; + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return false; } /* This check allows the one update to flag everything for cleanup. The check for full is only temporary while I test for bugs. */ if ((!strncmp(data_state,"goto_",5) - // || (!strcmp(data_state,"full")) - // || (!strcmp(data_state,"cleaned")) - )) { - char *query_detProcessedImfile = "UPDATE detProcessedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query_detProcessedImfile,data_state,det_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for detProcessedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return(false); - } - - char *query_detProcessedExp = "UPDATE detProcessedExp SET data_state = '%s'" - " WHERE det_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query_detProcessedExp,data_state,det_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for detProcessedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return(false); - } - - char *query_detNormalizedImfile = "UPDATE detNormalizedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; - if (!p_psDBRunQueryF(config->dbh, query_detNormalizedImfile,data_state,det_id,iteration)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for detNormalizedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return(false); - } - - char *query_detNormalizedStatImfile = "UPDATE detNormalizedStatImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; - if (!p_psDBRunQueryF(config->dbh, query_detNormalizedStatImfile,data_state,det_id,iteration)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for detNormalizedStatImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return(false); - } - - char *query_detNormalizedExp = "UPDATE detNormalizedExp SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; - if (!p_psDBRunQueryF(config->dbh, query_detNormalizedExp,data_state,det_id,iteration)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for detNormalizedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return(false); - } - - char *query_detResidImfile = "UPDATE detResidImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; - if (!p_psDBRunQueryF(config->dbh, query_detResidImfile,data_state,det_id,iteration)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for detResidImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return(false); - } - - char *query_detResidExp = "UPDATE detResidExp SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; - if (!p_psDBRunQueryF(config->dbh, query_detResidExp,data_state,det_id,iteration)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for detResidExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return(false); - } - - char *query_detStackedImfile = "UPDATE detStackedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; - if (!p_psDBRunQueryF(config->dbh, query_detStackedImfile,data_state,det_id,iteration)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for detStackedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); - return(false); - } + // || (!strcmp(data_state,"full")) + // || (!strcmp(data_state,"cleaned")) + )) { + char *query_detProcessedImfile = "UPDATE detProcessedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedImfile,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detProcessedExp = "UPDATE detProcessedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedExp,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedImfile = "UPDATE detNormalizedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedStatImfile = "UPDATE detNormalizedStatImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedStatImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedStatImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedExp = "UPDATE detNormalizedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidImfile = "UPDATE detResidImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidExp = "UPDATE detResidExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detStackedImfile = "UPDATE detStackedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detStackedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detStackedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } } /* End if */ - + } Index: branches/tap_branches/ippTools/src/dettool_normalizedexp.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_normalizedexp.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_normalizedexp.c (revision 27838) @@ -29,5 +29,5 @@ psString query = pxDataGet("dettool_tonormalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -89,5 +89,5 @@ psString query = pxDataGet("dettool_tonormalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -156,5 +156,5 @@ psString query = pxDataGet("dettool_normalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -225,5 +225,5 @@ psString query = pxDataGet("dettool_revertnormalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -278,5 +278,5 @@ psString query = pxDataGet("dettool_pendingcleanup_normalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -340,5 +340,5 @@ psString query = pxDataGet("dettool_donecleanup_normalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/dettool_normalizedimfile.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_normalizedimfile.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_normalizedimfile.c (revision 27838) @@ -30,5 +30,5 @@ psString query = pxDataGet("dettool_tonormalize.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -137,5 +137,5 @@ psString query = pxDataGet("dettool_normalizedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -205,5 +205,5 @@ psString query = pxDataGet("dettool_revertnormalizedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -260,5 +260,5 @@ psString query = pxDataGet("dettool_pendingcleanup_normalizedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -323,5 +323,5 @@ psString query = pxDataGet("dettool_donecleanup_normalizedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/dettool_normalizedstat.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_normalizedstat.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_normalizedstat.c (revision 27838) @@ -29,5 +29,5 @@ psString query = pxDataGet("dettool_tonormalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -117,5 +117,5 @@ psString query = pxDataGet("dettool_normalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -185,5 +185,5 @@ psString query = pxDataGet("dettool_revertnormalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -240,5 +240,5 @@ psString query = pxDataGet("dettool_pendingcleanup_normalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -303,5 +303,5 @@ psString query = pxDataGet("dettool_donecleanup_normalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/dettool_processedexp.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_processedexp.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_processedexp.c (revision 27838) @@ -29,5 +29,5 @@ psString query = pxDataGet("dettool_toprocessedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -101,5 +101,5 @@ psString query = pxDataGet("dettool_addprocessedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -235,5 +235,5 @@ psString query = pxDataGet("dettool_revertprocessedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -266,10 +266,36 @@ PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); - if (!setProcessedExpDataState(config, det_id, exp_id, data_state)) { - return false; - } - return true; + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id","det_id","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id","exp_id","=="); + + if (!setProcessedExpDataState(config, where, data_state)) { + return false; + } + psFree(where); + return true; +} + +bool updatestateprocessedMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + PXOPT_LOOKUP_S64(det_id,config->args, "-det_id",true,false); + PXOPT_LOOKUP_STR(data_state,config->args, "-data_state", true, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id","det_id","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id","exp_id","=="); + + if (!setProcessedExpDataState(config,where,data_state)) { + return(false); + } + if (!setProcessedImfileDataState(config,where,data_state)) { + return(false); + } + psFree(where); + return(true); } @@ -287,5 +313,5 @@ psString query = pxDataGet("dettool_pendingcleanup_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -349,5 +375,5 @@ psString query = pxDataGet("dettool_donecleanup_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/dettool_processedimfile.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_processedimfile.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_processedimfile.c (revision 27838) @@ -34,5 +34,5 @@ psString query = pxDataGet("dettool_toprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -180,5 +180,5 @@ psString query = pxDataGet("dettool_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -267,5 +267,5 @@ psString query = pxDataGet("dettool_revertprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -298,11 +298,16 @@ PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); - if (!setProcessedImfileDataState(config, det_id, exp_id, class_id, data_state)) { - return false; - } + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id", "det_id","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id", "exp_id","=="); + PXOPT_COPY_STR(config->args,where,"-class_id","class_id","=="); + + if (!setProcessedImfileDataState(config, where, data_state)) { + return false; + } + psFree(where); return true; } @@ -322,5 +327,5 @@ psString query = pxDataGet("dettool_pendingcleanup_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -351,10 +356,10 @@ if (!output) { psError(PS_ERR_UNKNOWN, false, "database error"); -/* fprintf(stderr,"WTF !output?\n"); */ +/* fprintf(stderr,"WTF !output?\n"); */ return false; } if (!psArrayLength(output)) { psTrace("dettool", PS_LOG_INFO, "no rows found"); -/* fprintf(stderr,"WTF no rows??\n"); */ +/* fprintf(stderr,"WTF no rows??\n"); */ psFree(output); return true; @@ -392,5 +397,5 @@ psString query = pxDataGet("dettool_donecleanup_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/dettool_residexp.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_residexp.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_residexp.c (revision 27838) @@ -35,5 +35,5 @@ psString query = pxDataGet("dettool_toresidexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -119,5 +119,5 @@ psString query = pxDataGet("dettool_toresidexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -202,5 +202,5 @@ psString query = pxDataGet("dettool_residexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -277,5 +277,5 @@ psString query = pxDataGet("dettool_revertresidexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -353,4 +353,24 @@ } +bool updatestateresidMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + PXOPT_LOOKUP_S64(det_id,config->args, "-det_id",true,false); + PXOPT_LOOKUP_STR(data_state,config->args, "-data_state", true, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id","det_id","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id","exp_id","=="); + + if (!setResidExpDataState(config,where,data_state)) { + return(false); + } + if (!setResidImfileDataState(config,where,data_state)) { + return(false); + } + psFree(where); + return(true); +} + bool pendingcleanup_residexpMode(pxConfig *config) { @@ -367,5 +387,5 @@ psString query = pxDataGet("dettool_pendingcleanup_residexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -430,5 +450,5 @@ psString query = pxDataGet("dettool_donecleanup_residexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/dettool_residimfile.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_residimfile.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_residimfile.c (revision 27838) @@ -29,5 +29,5 @@ psString query = pxDataGet("dettool_toresidimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -165,5 +165,5 @@ psString query = pxDataGet("dettool_residimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -253,5 +253,5 @@ psString query = pxDataGet("dettool_revertresidimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -284,9 +284,14 @@ PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); - if (!setResidImfileDataState(config, det_id, iteration, exp_id, class_id, data_state)) { + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id","det_id","=="); + PXOPT_COPY_S32(config->args,where,"-iteration", "iteration","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id","exp_id","=="); + PXOPT_COPY_STR(config->args,where,"-class_id","class_id","=="); + + if (!setResidImfileDataState(config, where, data_state)) { return false; } @@ -309,5 +314,5 @@ psString query = pxDataGet("dettool_pendingcleanup_residimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -373,5 +378,5 @@ psString query = pxDataGet("dettool_donecleanup_residimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/dettool_stack.c =================================================================== --- branches/tap_branches/ippTools/src/dettool_stack.c (revision 25900) +++ branches/tap_branches/ippTools/src/dettool_stack.c (revision 27838) @@ -31,5 +31,5 @@ psString query = pxDataGet("dettool_tostacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -220,5 +220,5 @@ psString query = pxDataGet("dettool_stacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -290,5 +290,5 @@ psString query = pxDataGet("dettool_revertstacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -345,5 +345,5 @@ psString query = pxDataGet("dettool_pendingcleanup_stacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -408,5 +408,5 @@ psString query = pxDataGet("dettool_donecleanup_stacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/difftool.c =================================================================== --- branches/tap_branches/ippTools/src/difftool.c (revision 25900) +++ branches/tap_branches/ippTools/src/difftool.c (revision 27838) @@ -43,4 +43,5 @@ static bool definewarpstackMode(pxConfig *config); static bool definewarpwarpMode(pxConfig *config); +static bool definestackstackMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -55,4 +56,7 @@ static bool topurgedskyfileMode(pxConfig *config); static bool toscrubbedskyfileMode(pxConfig *config); +static bool tofullskyfileMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); @@ -88,4 +92,5 @@ MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); + MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -94,7 +99,10 @@ MODECASE(DIFFTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(DIFFTOOL_MODE_IMPORTRUN, importrunMode); - MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); - MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); - MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); + MODECASE(DIFFTOOL_MODE_LISTRUN, listrunMode); + MODECASE(DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); default: @@ -132,5 +140,6 @@ PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-rset_eduction", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_S16(diff_mode, config->args, "-set_diff_mode", false, false); PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); @@ -153,4 +162,5 @@ exposure, false, + diff_mode, note ); @@ -186,10 +196,11 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-diff_id", "stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group","data_group", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -198,5 +209,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "diffRun", true); + bool result = pxUpdateRun(config, where, &query, "diffRun", "diff_id", "diffSkyfile", true); psFree(query); @@ -204,25 +215,4 @@ return result; - -#ifdef notdef - // required options - PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - - // Copy of my hacky work around from stacktool.c - if ((state)&&(diff_id)) { - // set detRun.state to state - return setdiffRunState(config, diff_id, state, false); - } - - if ((state)&&(label)) { - return setdiffRunStateByLabel(config, label, state); - } - - psError(PS_ERR_UNKNOWN, false, "Required options not found."); - - return false; -#endif } @@ -379,27 +369,23 @@ psString query = pxDataGet("difftool_inputskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString whereClause = NULL; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psStringCopy(""); if (psListLength(where->list)) { whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&whereClause, "\n AND "); + psStringPrepend(&whereClause, "\n WHERE "); } psFree(where); // Add condition to get only templates or only inputs + psString templateClause = psStringCopy(""); { - psString templateClause = NULL; if (template) { - psStringAppend(&templateClause, " %s", " template != 0"); + psStringAppend(&templateClause, "\n WHERE %s", " template != 0"); } else if (input) { - psStringAppend(&templateClause, " %s", " template = 0"); - } - if (templateClause) { - psStringAppend(&whereClause, "\n AND %s", templateClause); - } - psFree(templateClause); + psStringAppend(&templateClause, "\n WHERE %s", " template = 0"); + } } @@ -412,10 +398,12 @@ } - if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause, whereClause, templateClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(templateClause); psFree(whereClause); psFree(query); return false; } + psFree(templateClause); psFree(whereClause); psFree(query); @@ -469,5 +457,5 @@ psString query = pxDataGet("difftool_todiffskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -626,5 +614,5 @@ psString query = pxDataGet("difftool_completed_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -717,5 +705,7 @@ PXOPT_COPY_S64(config->args, where, "-magicked", "diffSkyfile.magicked", "=="); pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); - + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); if (!template) { @@ -725,5 +715,5 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "LIKE"); } else { PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); @@ -732,5 +722,5 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "LIKE"); } @@ -745,5 +735,5 @@ psString query = pxDataGet("difftool_skyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -756,9 +746,17 @@ psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); } else if (!all) { - psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); return false; } psFree(where); + if (pstamp_order) { + if (template) { + psStringAppend(&query, " ORDER BY rawTemplate.exp_id, diff_id DESC"); + } else { + psStringAppend(&query, " ORDER BY rawInput.exp_id, diff_id DESC"); + } + } + // treat limit == 0 as "no limit" if (limit) { @@ -816,27 +814,23 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "diffRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "diffRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Update state to 'new' + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + { - psString query = pxDataGet("difftool_revertdiffskyfile_update.sql"); + psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString query_updated = pxDataGet("difftool_revertdiffskyfile_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -845,4 +839,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); psFree(whereClause); } @@ -851,50 +846,21 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } return false; } psFree(query); + psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); } - // Delete product - { - psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); - } - psFree(where); - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } return true; @@ -914,5 +880,5 @@ if (magicked) { char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; - + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { psError(PS_ERR_UNKNOWN, false, @@ -923,5 +889,5 @@ else { char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64; - + if (!p_psDBRunQueryF(config->dbh, query, state, diff_id)) { psError(PS_ERR_UNKNOWN, false, @@ -930,5 +896,5 @@ } } - + return true; } @@ -948,5 +914,5 @@ if (!p_psDBRunQueryF(config->dbh,query,state,label)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for label %s", label); + "failed to change state for label %s", label); return(false); } @@ -986,4 +952,18 @@ return false; } + psS16 diff_mode = IPP_DIFF_MODE_UNDEFINED; + if ((input_warp_id != PS_MAX_S64) && (template_warp_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_WARP_WARP; + } + else if ((input_warp_id != PS_MAX_S64) && (template_stack_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_WARP_STACK; + } + else if ((input_stack_id != PS_MAX_S64) && (template_warp_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_STACK_WARP; + } + else if ((input_stack_id != PS_MAX_S64) && (template_stack_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_STACK_STACK; + } + // default @@ -1005,4 +985,5 @@ false, 0, // magicked + diff_mode, // diff_mode note ); @@ -1110,5 +1091,5 @@ psArray *list = psArrayAllocEmpty(16); // List of runs, to print - + // Populated Run will generate the diff_mode value if (!populatedrun(list, workdir, skycell_id, tess_id, label, data_group ? data_group : label, dist_group, reduction, note, input_warp_id ? input_warp_id : PS_MAX_S64, @@ -1138,14 +1119,21 @@ psMetadata *expWhere = psMetadataAlloc(); - psMetadata *warpWhere = psMetadataAlloc(); + psMetadata *warp1Where = psMetadataAlloc(); // First set of restrictions on warp + psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp psMetadata *stackWhere = psMetadataAlloc(); PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "=="); PXOPT_COPY_STR(config->args, expWhere, "-filter", "filter", "=="); - PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpSkyfile.skycell_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-tess_id", "warpRun.tess_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "=="); - PXOPT_COPY_F32(config->args, warpWhere, "-good_frac", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, expWhere, "-comment", "comment", "LIKE"); + PXOPT_COPY_S64(config->args, warp1Where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-warp_label", "warpRun.label", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "=="); + PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "=="); + PXOPT_COPY_F32(config->args, warp2Where, "-good_frac", "warpSkyfile.good_frac", ">="); PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "=="); @@ -1167,9 +1155,10 @@ psString query = pxDataGet("difftool_definewarpstack_part1.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString warpQuery = NULL; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString warp1Query = NULL; + psString warp2Query = NULL; psString stackQuery = NULL; psString expQuery = NULL; @@ -1183,17 +1172,20 @@ } psFree(expWhere); - if (psListLength(warpWhere->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL); - psStringAppend(&warpQuery, "\n AND %s", whereClause); + if (psListLength(warp1Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(warp1Where, NULL); + psStringAppend(&warp1Query, "\n AND %s", whereClause); psFree(whereClause); } else { - warpQuery = psStringCopy("\n"); - } - psFree(warpWhere); - - if (!available) { - // diff what's available, even if warp run has some faults and is incomplete - psStringAppend(&warpQuery, " AND warpRun.state = 'full'"); - } + warp1Query = psStringCopy("\n"); + } + psFree(warp1Where); + if (psListLength(warp2Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(warp2Where, NULL); + psStringAppend(&warp2Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + warp2Query = psStringCopy("\n"); + } + psFree(warp2Where); // don't queue for exposures that have already been diff'd unless requested @@ -1214,5 +1206,5 @@ psFree(stackWhere); - psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery); + psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery); if (!psDBTransaction(config->dbh)) { @@ -1221,5 +1213,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, warpQuery, expQuery, diffQuery)) { + if (!p_psDBRunQueryF(config->dbh, query, warp1Query, warp2Query, expQuery, diffQuery)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1281,4 +1273,5 @@ return false; } + psFree(warp1Query); psFree(query); query = NULL; @@ -1312,5 +1305,5 @@ if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1323,5 +1316,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1334,5 +1327,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "skycell_count not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1345,5 +1338,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "tess_id not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1353,8 +1346,9 @@ return false; } + psString filter = psMetadataLookupStr(&mdok, row, "filter"); if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "filter not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1364,7 +1358,23 @@ return false; } - if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warpQuery)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + + psString warp_data_group = psMetadataLookupStr(&mdok, row, "data_group"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "warp data_group not found"); + psFree(warp2Query); + psFree(stackQuery); + psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (warp_data_group) { + data_group = warp_data_group; + } + + if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warp2Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1377,5 +1387,5 @@ if (num == 0) { - psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id); + psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for warp_id %" PRId64, warp_id); continue; } @@ -1383,5 +1393,5 @@ if (!available && (num != skycell_count)) { psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64 - " need %" PRId64, num, warp_id, skycell_count); + " but need %" PRId64, num, warp_id, skycell_count); continue; } @@ -1402,4 +1412,5 @@ true, // exposure 0, // magicked + IPP_DIFF_MODE_WARP_STACK, note ); @@ -1418,5 +1429,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1432,5 +1443,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1447,5 +1458,5 @@ psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, run->diff_id); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1461,5 +1472,5 @@ } psFree(output); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1549,5 +1560,4 @@ // Additional controls PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); - PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); // Settings to apply to defined run @@ -1574,5 +1584,5 @@ psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL if (!tempCreate) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1597,5 +1607,5 @@ psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table if (!tempInsert) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1629,5 +1639,5 @@ psString select = pxDataGet("difftool_definewarpwarp_select.sql"); if (!select) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1642,18 +1652,11 @@ if (psListLength(selectWhere->list)) { psString new = psDBGenerateWhereConditionSQL(selectWhere, NULL); - psStringAppend(&whereClause, "\n%s %s", whereClause ? "AND" : "WHERE", new); + psStringAppend(&whereClause, "\nAND %s", new); psFree(new); } psFree(selectWhere); - if (!available) { - psStringAppend(&whereClause, - "\n%s inputWarpRun.state = 'full'" - "AND templateWarpRun.state = 'full'", - whereClause ? "AND" : "WHERE"); - } - if (!rerun) { - psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE"); + psStringAppend(&whereClause, "\nAND diffs.diff_id IS NULL"); } @@ -1732,4 +1735,9 @@ const char *template = psMetadataLookupStr(NULL, row, "template_warp_id"); const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); + psString input_data_group = psMetadataLookupStr(NULL, row, "input_data_group"); + if (input_data_group) { + data_group = input_data_group; + } + if (!input_id || !template || !tess_id) { psError(PXTOOLS_ERR_PROG, false, "Identifiers not found"); @@ -1742,7 +1750,20 @@ return false; } - - diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, data_group ? data_group : label, dist_group, reduction, NULL, registered, - tess_id, true, true, false, note); // Run to insert + + diffRunRow *run = diffRunRowAlloc(0, + "reg", + workdir, + label, + data_group ? data_group : label, + dist_group, + reduction, + NULL, // dvodb + registered, + tess_id, + true, // bothways + true, // exposure + false, // magicked + IPP_DIFF_MODE_WARP_WARP, + note); // Run to insert if (!diffRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1816,4 +1837,303 @@ } +static bool definestackstackMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *stack1Where = psMetadataAlloc(); + psMetadata *stack2Where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, stack1Where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-input_label", "stackRun.label","=="); + PXOPT_COPY_STR(config->args, stack2Where, "-template_label", "stackRun.label","=="); + PXOPT_COPY_F32(config->args, stack1Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); + PXOPT_COPY_F32(config->args, stack2Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(newTemplates,config->args,"-new-templates", false); + + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + if (!(label)) { + PXOPT_LOOKUP_STR(label,config->args, "-input_label",true,false); + } + + // Organize the config information into queries. + psString stack1Query = NULL; + psString stack2Query = NULL; + + if (psListLength(stack1Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(stack1Where, NULL); + psStringAppend(&stack1Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + stack1Query = psStringCopy("\n"); + } + psFree(stack1Where); + + if (psListLength(stack2Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(stack2Where, NULL); + psStringAppend(&stack2Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + stack2Query = psStringCopy("\n"); + } + psFree(stack2Where); + + // don't queue for stacks that have already been diffed unless requested + psString diffQuery = NULL; + if (! (reRun || newTemplates) ) { + psStringAppend(&diffQuery, "\n AND diffExp.diff_id IS NULL"); + } else { + diffQuery = psStringCopy("\n"); + } + + // find the distinct set of data_groups and filters + psString query = pxDataGet("difftool_definestackstack_part0.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psTrace("difftool",1,query,stack2Query,stack1Query,diffQuery,stack1Query); +/* exit(10); */ + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, stack2Query, stack1Query, diffQuery, stack1Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + query = pxDataGet("difftool_definestackstack_part1.sql"); + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRunTemp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs defined, to print + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Output row from query + bool mdok; // Status of MD lookup + + psString tess_id = psMetadataLookupStr(&mdok,row,"INPUT_tess_id"); + psString this_data_group = psMetadataLookupStr(&mdok,row,"INPUT_data_group"); + psString this_dist_group = psMetadataLookupStr(&mdok,row,"INPUT_dist_group"); + psString this_label = psMetadataLookupStr(&mdok,row,"INPUT_label"); + + psString this_stack1Query = psStringCopy(stack1Query); + + psString thisWhere = psDBGenerateWhereConditionSQL(row,NULL); + psStringSubstitute(&thisWhere,"stackRun.","INPUT_"); + psStringAppend(&this_stack1Query,"\n AND %s", thisWhere); + psFree(thisWhere); + + psTrace("difftool",1, query,stack2Query,this_stack1Query,diffQuery,this_stack1Query); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, stack2Query, this_stack1Query, diffQuery, this_stack1Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(this_stack1Query); + + psArray *diff_id_output = p_psDBFetchResult(config->dbh); + if (!diff_id_output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(diff_id_output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(diff_id_output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + // ok we've got one create the diffRun + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + label ? label : this_label, + data_group ? data_group : this_data_group, + dist_group ? dist_group : this_dist_group, + reduction, + NULL, // dvodb + registered, + tess_id, + false, // bothways + false, // exposure + 0, // magicked + IPP_DIFF_MODE_STACK_STACK, // diff_mode + note + ); + // Commit to database + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } +/* diffRunPrintObject(stdout,run,1); */ + run->diff_id = psDBLastInsertID(config->dbh); + for (long j = 0; j < diff_id_output->n; j++) { + psMetadata *input_row = diff_id_output->data[j]; // Output row from query + bool mdok; + psString in_skycell_id = psMetadataLookupStr(&mdok,input_row,"skycell_id"); + psS64 in_input_stack_id = psMetadataLookupS64(&mdok,input_row,"stack_id"); + psS64 in_template_stack_id = psMetadataLookupS64(&mdok,input_row,"max_stack_id"); + psString in_tess_id = psMetadataLookupStr(&mdok,input_row,"tess_id"); + psTrace("difftool",1,"%s %" PRId64 " %" PRId64 " %s\n", + in_skycell_id, in_input_stack_id, in_template_stack_id, in_tess_id); + diffInputSkyfileRow *input = diffInputSkyfileRowAlloc( + run->diff_id, // ID + in_skycell_id, + PS_MAX_S64, // warp1_id -> NULL + in_input_stack_id, // stack1 + PS_MAX_S64, // warp2_id -> NULL + in_template_stack_id, // stack2 + in_tess_id, + 0 // diff_skyfile_id + ); + // Commit to database the input + if (!diffInputSkyfileInsertObject(config->dbh, input)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(input); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + +/* diffInputSkyfilePrintObject(stdout,input,1); */ + psFree(input); + } + + if (!setdiffRunState(config, run->diff_id, "new", false)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, run->diff_id); + psFree(stack1Query); + psFree(stack2Query); + psFree(diffQuery); + psFree(run); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psArrayAdd(list, list->n, run); + psFree(run); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(query); + psFree(stack1Query); + psFree(stack2Query); + psFree(diffQuery); + psFree(output); + psFree(list); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + return(true); +} static bool pendingcleanuprunMode(pxConfig *config) @@ -1825,16 +2145,24 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); psString query = pxDataGet("difftool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // pxDataGet/psStringSubstitute workaround + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereClause, " AND "); + psStringSubstitute(&query,whereClause,"@INNERCONSTRAINT@"); psFree(whereClause); + } + else { + psStringSubstitute(&query,NULL,"@INNERCONSTRAINT@"); } psFree(where); @@ -1846,6 +2174,7 @@ psFree(limitString); } - // fprintf(stderr,"%s",query); + + //fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1895,5 +2224,5 @@ psString query = pxDataGet("difftool_pendingcleanupskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1955,5 +2284,5 @@ psString query = pxDataGet("difftool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -2010,4 +2339,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) { @@ -2027,4 +2357,11 @@ PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + + psS64 magicked = 0; + if (!strcmp(data_state, "full")) { + PXOPT_LOOKUP_S64(set_magicked, config->args, "-magicked", 0, false); + magicked = set_magicked; + } psString query = pxDataGet("difftool_change_skyfile_data_state.sql"); @@ -2035,7 +2372,18 @@ } + psString set_magicked_skyfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (magicked != 0 && !strcmp(data_state, "full")) { + psStringAppend(&set_magicked_skyfile, "\n , diffSkyfile.magicked = %" PRId64, magicked); + psStringAppend(&set_magicked_run, "\n , diffRun.magicked = %" PRId64, magicked); + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + psStringAppend(&set_magicked_skyfile, "\n, diffSkyfile.magicked = IF(diffSkyfile.magicked = 0, 0, -1)"); + psStringAppend(&set_magicked_run, "\n, diffRun.magicked = IF(diffRun.magicked = 0, 0, -1)"); + } + // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c - if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, skycell_id)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, diff_id, skycell_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); // rollback @@ -2044,10 +2392,12 @@ } psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_skyfile); return(false); } + psFree(set_magicked_skyfile); psFree(query); query = pxDataGet("difftool_change_run_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, diff_id, data_state)) { // rollback if (!psDBRollback(config->dbh)) { @@ -2055,4 +2405,5 @@ } psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_run); return(false); } @@ -2060,7 +2411,9 @@ if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_run); return(false); } - + psFree(set_magicked_run); + return(true); } @@ -2074,4 +2427,7 @@ static bool toscrubbedskyfileMode(pxConfig *config) { return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); +} +static bool tofullskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "full", "update"); } @@ -2115,5 +2471,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -2252,2 +2608,148 @@ return true; } + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "diffRun.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + if (!template) { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "LIKE"); + } else { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "LIKE"); + } + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "diffRun"); + pxspaceAddWhere(config, &where2, template ? "rawTemplate" : "rawInput"); + psString query = pxDataGet("difftool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (where2) { + psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + if (template) { + psStringAppend(&query, " ORDER BY rawTemplate.exp_id, diff_id DESC"); + } else { + psStringAppend(&query, " ORDER BY rawInput.exp_id, diff_id DESC"); + } + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("difftool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , diffRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} Index: branches/tap_branches/ippTools/src/difftool.h =================================================================== --- branches/tap_branches/ippTools/src/difftool.h (revision 25900) +++ branches/tap_branches/ippTools/src/difftool.h (revision 27838) @@ -37,4 +37,5 @@ DIFFTOOL_MODE_DEFINEWARPSTACK, DIFFTOOL_MODE_DEFINEWARPWARP, + DIFFTOOL_MODE_DEFINESTACKSTACK, DIFFTOOL_MODE_PENDINGCLEANUPRUN, DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, @@ -45,6 +46,8 @@ DIFFTOOL_MODE_TOCLEANEDSKYFILE, DIFFTOOL_MODE_TOPURGEDSKYFILE, - DIFFTOOL_MODE_TOSCRUBBEDSKYFILE - + DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, + DIFFTOOL_MODE_TOFULLSKYFILE, + DIFFTOOL_MODE_LISTRUN, + DIFFTOOL_MODE_SETSKYFILETOUPDATE, } difftoolMode; Index: branches/tap_branches/ippTools/src/difftoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/difftoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/difftoolConfig.c (revision 27838) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -56,4 +56,5 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddS16(definerunArgs, PS_LIST_TAIL, "-set_diff_mode", 0, "specify type of difference (WW=1,WS=2,SW=3,SS=4)", 0); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -64,8 +65,10 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define by label instead of diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "define by data_group instead of diff ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_diff_mode", 0, "specify type of difference (WW=1,WS=2,SW=3,SS=4)", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); @@ -122,5 +125,5 @@ psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); - psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", false); + psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); // -advance @@ -144,16 +147,44 @@ psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL); psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); pxmagicAddArguments(diffskyfileArgs); pxspaceAddArguments(diffskyfileArgs); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", NULL); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-template", 0, "apply exposure args to template of bothways diff", false); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL); + pxmagicAddArguments(listrunArgs); + pxspaceAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertdiffskyfile psMetadata *revertdiffskyfileArgs = psMetadataAlloc(); psMetadataAddS64(revertdiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); - psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); psMetadataAddS16(revertdiffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddBool(revertdiffskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); @@ -180,6 +211,8 @@ psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-comment", 0, "search by comment (LIKE)", NULL); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); @@ -192,5 +225,5 @@ psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); - psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if no stacks available for some skycells", false); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false); @@ -217,5 +250,4 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); - psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false); @@ -250,7 +282,30 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + // -definestackstack + psMetadata *definestackstackArgs = psMetadataAlloc(); + // stack id and exp id searches seem less useful here + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_label", 0, "search by stack label for input", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-template_label", 0, "search by stack label for template", NULL); + psMetadataAddF32(definestackstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definestackstackArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if no stacks available for some skycells", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false); + + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); - psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list runs for cleanup with specified label", NULL); psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -271,5 +326,6 @@ psMetadata *updatediffskyfileArgs = psMetadataAlloc(); psMetadataAddS64(updatediffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); - psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); // -tocleanedskyfile @@ -287,4 +343,18 @@ psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -tofullskyfile + psMetadata *tofullskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); + + // -setskyfiletoupdate + psMetadata *setskyfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setskyfiletoupdateArgs, PS_LIST_TAIL, "-diff_id", 0, "search by difftool ID (required)", 0); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "new value for diffRun.label", NULL); + + // -exportrun @@ -314,12 +384,16 @@ PXOPT_ADD_MODE("-advance", "", DIFFTOOL_MODE_ADVANCE, advanceArgs); PXOPT_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); + PXOPT_ADD_MODE("-listrun", "", DIFFTOOL_MODE_LISTRUN, listrunArgs); PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); PXOPT_ADD_MODE("-definewarpstack", "", DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackArgs); PXOPT_ADD_MODE("-definewarpwarp", "", DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpArgs); + PXOPT_ADD_MODE("-definestackstack", "", DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackArgs); PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", DIFFTOOL_MODE_DONECLEANUP, donecleanupArgs); PXOPT_ADD_MODE("-updatediffskyfile", "update fault code for a diffskyfile", DIFFTOOL_MODE_UPDATEDIFFSKYFILE, updatediffskyfileArgs); + PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", DIFFTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", DIFFTOOL_MODE_IMPORTRUN, importrunArgs); @@ -328,5 +402,6 @@ PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); - + PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { psError(PS_ERR_UNKNOWN, true, "option parsing failed"); Index: branches/tap_branches/ippTools/src/disttool.c =================================================================== --- branches/tap_branches/ippTools/src/disttool.c (revision 25900) +++ branches/tap_branches/ippTools/src/disttool.c (revision 27838) @@ -42,12 +42,15 @@ static bool addfilesetMode(pxConfig *config); static bool revertfilesetMode(pxConfig *config); +static bool updatefilesetMode(pxConfig *config); static bool queuercrunMode(pxConfig *config); static bool updatercrunMode(pxConfig *config); static bool revertrcrunMode(pxConfig *config); static bool pendingdestMode(pxConfig *config); +static bool pendingcleanupMode(pxConfig *config); +static bool listfilesetsMode(pxConfig *config); static bool definetargetMode(pxConfig *config); static bool updatetargetMode(pxConfig *config); -static bool listtargetMode(pxConfig *config); +static bool listtargetsMode(pxConfig *config); static bool definedestinationMode(pxConfig *config); @@ -86,6 +89,9 @@ MODECASE(DISTTOOL_MODE_TOADVANCE, toadvanceMode); MODECASE(DISTTOOL_MODE_PENDINGFILESET, pendingfilesetMode); + MODECASE(DISTTOOL_MODE_PENDINGCLEANUP, pendingcleanupMode); MODECASE(DISTTOOL_MODE_ADDFILESET, addfilesetMode); MODECASE(DISTTOOL_MODE_REVERTFILESET, revertfilesetMode); + MODECASE(DISTTOOL_MODE_LISTFILESETS, listfilesetsMode); + MODECASE(DISTTOOL_MODE_UPDATEFILESET, updatefilesetMode); MODECASE(DISTTOOL_MODE_QUEUERCRUN, queuercrunMode); MODECASE(DISTTOOL_MODE_UPDATERCRUN, updatercrunMode); @@ -94,5 +100,5 @@ MODECASE(DISTTOOL_MODE_DEFINETARGET, definetargetMode); MODECASE(DISTTOOL_MODE_UPDATETARGET, updatetargetMode); - MODECASE(DISTTOOL_MODE_LISTTARGET, listtargetMode); + MODECASE(DISTTOOL_MODE_LISTTARGETS, listtargetsMode); MODECASE(DISTTOOL_MODE_DEFINEDESTINATION, definedestinationMode); MODECASE(DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationMode); @@ -131,13 +137,25 @@ // optional - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); + PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false); PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); PXOPT_LOOKUP_STR(set_note, config->args, "-set_note", false, false); PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + if (use_alternate) { + if (strcmp(stage, "raw")) { + psError(PXTOOLS_ERR_SYS, true, "alternate inputs only supported for raw stage"); + return false; + } + if (no_magic) { + psError(PXTOOLS_ERR_SYS, true, "no_magic forbidden with alternate inputs"); + return false; + } + } // select arguments @@ -155,19 +173,24 @@ psString runJoinStr = NULL; if (!strcmp(stage, "raw")) { - magicRunType = "rawExp"; - runJoinStr = "rawExp.exp_id"; + if (! use_alternate ) { + magicRunType = "rawExp"; + runJoinStr = "rawExp.exp_id"; + } else { + magicRunType = "camRun"; + runJoinStr = "camRun.exp_id"; + } query = pxDataGet("disttool_definebyquery_raw.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; } - // for raw stage we select by chipRun.label because raw labels typically aren't set + // for raw stage we select by camRun.label and dist_group because rawExp doesn't have those columns if (label) { - psStringAppend(&query, " AND (chipRun.label = '%s')", label); + psStringAppend(&query, " AND (magicDSRun.label = '%s')", label); } if (dist_group) { - psStringAppend(&query, " AND (chipRun.dist_group = '%s')", dist_group); + psStringAppend(&query, " AND (camRun.dist_group = '%s')", dist_group); } } else if (!strcmp(stage, "chip")) { @@ -176,5 +199,5 @@ query = pxDataGet("disttool_definebyquery_chip.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -192,5 +215,5 @@ query = pxDataGet("disttool_definebyquery_camera.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -207,5 +230,5 @@ query = pxDataGet("disttool_definebyquery_fake.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -225,5 +248,5 @@ query = pxDataGet("disttool_definebyquery_warp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -242,5 +265,5 @@ query = pxDataGet("disttool_definebyquery_diff.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -258,5 +281,5 @@ query = pxDataGet("disttool_definebyquery_stack.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -271,8 +294,33 @@ // stack stage doesn't require magic no_magic = true; + } else if (!strcmp(stage, "SSdiff")) { + magicRunType = "diffRun"; + runJoinStr = "diffRun.diff_id"; + query = pxDataGet("disttool_definebyquery_SSdiff.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (diffRun.label = '%s') ", label); + } + if (dist_group) { + psStringAppend(&query, " AND (diffRun.dist_group = '%s') ", dist_group); + } + + no_magic = true; } else { psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); psFree(where); return false; + } + + if (!strcmp(stage, "raw")) { + if (!no_magic) { + psStringAppend(&query, " AND (magicDSRun.re_place = %d)", !use_alternate); + psStringAppend(&query, " AND (camRun.state = 'full')"); + } } @@ -287,17 +335,5 @@ if (!no_magic) { - psStringAppend(&query, " AND (%s.magicked)", magicRunType); - - // is selecting by magic_ds_id really interesting? - if (magic_ds_id) { - if (!runJoinStr) { - psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); - psFree(query); - return false; - } - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" - " AND magicDSRun.stage_id = %s", runJoinStr); - psStringAppend(&query, " AND (magicDSRun.state = 'full' AND magicDSRun.re_place AND (magic_ds_id = %" PRId64 "))", magic_ds_id); - } + psStringAppend(&query, " AND (%s.magicked > 0)", magicRunType); } @@ -307,9 +343,9 @@ psFree(limitString); } - + psTrace("disttool",2,query,joinHook ? joinHook : ""); if (!p_psDBRunQueryF(config->dbh, query, joinHook ? joinHook : "")) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; } psFree(query); @@ -328,5 +364,5 @@ - if (dry_run) { + if (pretend) { if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", true)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -352,8 +388,9 @@ psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); psString stage_label = psMetadataLookupStr(NULL, md, "label"); + psString stage_data_group = psMetadataLookupStr(NULL, md, "data_group"); bool clean = psMetadataLookupBool(NULL, md, "clean"); psString outroot = NULL; - psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage); + psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage); psString new_label; @@ -363,4 +400,11 @@ new_label = stage_label; } + psString new_data_group; + if (set_data_group != NULL) { + new_data_group = set_data_group; + } else { + new_data_group = stage_data_group; + } + distRunRow *row = distRunRowAlloc( 0, // dist_id @@ -371,10 +415,13 @@ new_label, outroot, + NULL, // outdir clean, no_magic, + use_alternate, "new", - NULL, // time_stamp + NULL, // time_stamp 0, // fault - set_note // note does not propagate + new_data_group, + set_note // note does not propagate ); @@ -419,10 +466,13 @@ PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; - PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "distRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "distRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "distTarget.dist_group", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -430,25 +480,40 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - - if ((!state) && (!label) && (!fault)) { - psError(PXTOOLS_ERR_DATA, false, "parameters (-fault or -set_state or -set_label) are required"); - psFree(where); - return false; - } - - psString query = psStringCopy("UPDATE distRun SET time_stamp = UTC_TIMESTAMP()"); - + PXOPT_LOOKUP_STR(set_note, config->args, "-set_note", false, false); + + if ((!state) && (!label) && (!fault) &&(!data_group)) { + psError(PXTOOLS_ERR_CONFIG, false, "parameters (-fault or -set_state or -set_label -set_data_group) are required"); + psFree(where); + return false; + } + + psString query = psStringCopy("UPDATE distRun join distTarget using(target_id, stage) SET distRun.time_stamp = UTC_TIMESTAMP()"); + + if (outdir) { + psStringAppend(&query, " , distRun.outdir = '%s'", outdir); + } if (state) { - psStringAppend(&query, " , state = '%s'", state); + psStringAppend(&query, " , distRun.state = '%s'", state); } if (label) { - psStringAppend(&query, " , label = '%s'", label); + psStringAppend(&query, " , distRun.label = '%s'", label); } if (fault) { - psStringAppend(&query, " , fault = %d", fault); - } + psStringAppend(&query, " , distRun.fault = %d", fault); + } + + if (data_group) { + psStringAppend(&query, " , distRun.data_group = '%s'", data_group); + } + + if (set_note) { + psStringAppend(&query, " , distRun.note = '%s'", set_note); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -481,5 +546,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -487,5 +552,5 @@ psString query = pxDataGet("disttool_revertrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -522,5 +587,5 @@ PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs(config, where, "-label", "distRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "distComponent.fault", "=="); @@ -530,5 +595,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -536,5 +601,5 @@ psString query = pxDataGet("disttool_revertcomponent.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -568,5 +633,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); @@ -578,5 +643,5 @@ psString query = pxDataGet(queryFile); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement from %s", queryFile); return false; } @@ -652,7 +717,8 @@ PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", require_fileinfo, false); PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", require_fileinfo, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", require_fileinfo, false); PXOPT_LOOKUP_STR(name, config->args, "-name", require_fileinfo, false); - if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", name, fault)) { + if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", outdir, name, fault)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -676,5 +742,5 @@ psString query = pxDataGet("disttool_toadvance.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -747,5 +813,5 @@ psString query = pxDataGet("disttool_processedcomponent.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -821,5 +887,5 @@ psString query = pxDataGet("disttool_pendingfileset.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -891,5 +957,5 @@ PXOPT_LOOKUP_STR(name, config->args, "-name", fault == 0, false); - if (!rcDSFilesetInsert(config->dbh, + if (!rcDSFilesetInsert(config->dbh, 0, // fs_id dist_id, @@ -904,4 +970,5 @@ return true; } + static bool revertfilesetMode(pxConfig *config) { @@ -920,5 +987,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -926,5 +993,5 @@ psString query = pxDataGet("disttool_revertfileset.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -968,5 +1035,5 @@ psString query = pxDataGet("disttool_pendingdest.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1047,5 +1114,5 @@ psString query = pxDataGet("disttool_queuercrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1093,5 +1160,5 @@ // We either need rc_id or (dest_id and fs_name) to identifiy the rcRun if ((!rc_id) && !(dest_id && fs_name)) { - psError(PXTOOLS_ERR_DATA, true, "either -rc_id or (-fs_name and -dest_id) are required"); + psError(PXTOOLS_ERR_CONFIG, true, "either -rc_id or (-fs_name and -dest_id) are required"); return false; } @@ -1105,5 +1172,5 @@ // this can't happen because we checked above psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -1115,5 +1182,5 @@ if (!state && (fault < 0)) { - psError(PXTOOLS_ERR_DATA, false, "parameters (-fault or -set_state) are required"); + psError(PXTOOLS_ERR_CONFIG, false, "parameters (-fault or -set_state) are required"); psFree(where); return false; @@ -1122,5 +1189,5 @@ psString query = pxDataGet("disttool_updatercrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1179,5 +1246,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -1185,5 +1252,5 @@ psString query = pxDataGet("disttool_revertrcrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1238,5 +1305,5 @@ comment ); - + if (!row) { psError(PS_ERR_UNKNOWN, false, "failed to allocate distTarget object"); @@ -1298,5 +1365,5 @@ } -static bool listtargetMode(pxConfig *config) +static bool listtargetsMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -1404,5 +1471,5 @@ state ? state : "enabled" ); - + if (!row) { psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDestination object"); @@ -1441,5 +1508,5 @@ if (!(state || last_fileset)) { psError(PS_ERR_UNKNOWN, true, "one or more of -set_state or -set_last_fileset is required"); -# else +# else if (!state) { #endif @@ -1538,5 +1605,8 @@ PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + // if stage is all don't add it to the query (match all stages) + if (stage && strcmp(stage, "all")) { + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + } psString query = pxDataGet("disttool_defineinterest.sql"); @@ -1598,4 +1668,17 @@ PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + if (full && clean) { + psError(PS_ERR_UNKNOWN, true, "-full and -clean makes no sense, chose one or the other"); + psFree(where); + return false; + } + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); @@ -1605,6 +1688,5 @@ return false; } - psString query = NULL; - psStringAppend(&query, "UPDATE rcInterest SET state = '%s'", state); + psString query = pxDataGet("disttool_updateinterest.sql"); if (psListLength(where->list)) { @@ -1620,18 +1702,173 @@ psFree(where); + if (clean) { + psStringAppend(&query, " AND distTarget.clean"); + } else { + psStringAppend(&query, " AND NOT distTarget.clean"); + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psS64 numUpdated = psDBAffectedRows(config->dbh); + printf("updated %" PRId64 " interests\n", numUpdated); + + return true; +} +static bool listinterestsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "rcInterest.state", "=="); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (clean && full) { + psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full"); + return false; + } + + psString query = pxDataGet("disttool_listinterests.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + if (clean) { + psStringAppend(&query, " AND (clean)"); + } else if (full) { + psStringAppend(&query, " AND (!clean)"); + } + } else if (clean) { + psStringAppend(&query, " WHERE clean"); + } else if (full) { + psStringAppend(&query, " WHERE !clean"); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } psFree(query); - return true; -} -static bool listinterestsMode(pxConfig *config) + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "rcInterest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool pendingcleanupMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + psString query = pxDataGet("disttool_pendingcleanup.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + } else if (!all) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters or -all are required"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "distToCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool listfilesetsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); @@ -1646,28 +1883,20 @@ PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - if (clean && full) { - psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full"); - return false; - } - - psString query = pxDataGet("disttool_listinterests.sql"); - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - if (clean) { - psStringAppend(&query, " AND (clean)"); - } else if (full) { - psStringAppend(&query, " AND (!clean)"); - } - } else if (clean) { - psStringAppend(&query, " WHERE clean"); - } else if (full) { - psStringAppend(&query, " WHERE !clean"); - } + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("disttool_listfilesets.sql"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); psFree(where); @@ -1700,5 +1929,5 @@ } - if (!ippdbPrintMetadatas(stdout, output, "rcInterest", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "distFilesets", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -1710,2 +1939,49 @@ return true; } +static bool updatefilesetMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + // We don't use PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); here + // because we want -fault 0 to work + bool gotFault = false; + psS16 fault = psMetadataLookupS16(&gotFault, config->args, "-fault"); + + if ((!state) && (!gotFault)) { + psError(PXTOOLS_ERR_CONFIG, true, "parameters (-fault or -set_state) is required"); + psFree(where); + return false; + } + + psString query = psStringCopy("UPDATE rcDSFileset SET "); + + if (state) { + psStringAppend(&query, " state = '%s'", state); + } + + if (gotFault) { + psStringAppend(&query, "%s fault = %d", state ? ", " : "", fault); + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} Index: branches/tap_branches/ippTools/src/disttool.h =================================================================== --- branches/tap_branches/ippTools/src/disttool.h (revision 25900) +++ branches/tap_branches/ippTools/src/disttool.h (revision 27838) @@ -33,7 +33,10 @@ DISTTOOL_MODE_PROCESSEDCOMPONENT, DISTTOOL_MODE_TOADVANCE, + DISTTOOL_MODE_PENDINGCLEANUP, DISTTOOL_MODE_PENDINGFILESET, DISTTOOL_MODE_ADDFILESET, DISTTOOL_MODE_REVERTFILESET, + DISTTOOL_MODE_LISTFILESETS, + DISTTOOL_MODE_UPDATEFILESET, DISTTOOL_MODE_QUEUERCRUN, DISTTOOL_MODE_UPDATERCRUN, @@ -44,5 +47,5 @@ DISTTOOL_MODE_DEFINETARGET, DISTTOOL_MODE_UPDATETARGET, - DISTTOOL_MODE_LISTTARGET, + DISTTOOL_MODE_LISTTARGETS, DISTTOOL_MODE_DEFINEINTEREST, DISTTOOL_MODE_UPDATEINTEREST, Index: branches/tap_branches/ippTools/src/disttoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/disttoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/disttoolConfig.c (revision 27838) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psError(psErrorCodeLast(), false, "Can't find site configuration!\n"); psFree(config); return NULL; @@ -49,14 +49,15 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group for run", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "select by dist_group", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be selected", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); // select args - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define chip_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by run label", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dist_group", 0, "select by dist_group", NULL); @@ -70,9 +71,13 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_outdir", 0, "new value for outdir", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "value for stage", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "value for state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "limit updates to label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "limit updates to data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "limit updates to data_group", NULL); psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); - // -revertrun psMetadata *revertrunArgs = psMetadataAlloc(); @@ -93,4 +98,13 @@ psMetadataAddBool(pendingcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -pendingcleanup + psMetadata *pendingcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(pendingcleanupArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); + psMetadataAddStr(pendingcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); + psMetadataAddU64(pendingcleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(pendingcleanupArgs, PS_LIST_TAIL, "-all", 0, "list all runs pending cleanup", NULL); + // -addprocessedcomponent @@ -98,4 +112,5 @@ psMetadataAddS64(addprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL); + psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory", NULL); psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0); @@ -110,5 +125,5 @@ psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); - psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); psMetadataAddS16(revertcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); psMetadataAddBool(revertcomponentArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); @@ -142,4 +157,11 @@ psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); psMetadataAddS32(addfilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + + // -updatefileset + psMetadata *updatefilesetArgs = psMetadataAlloc(); + psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); +// psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-set_state",0, "new value for state", NULL); + psMetadataAddS32(updatefilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); // -pendingdest @@ -162,4 +184,21 @@ psMetadataAddBool(revertfilesetArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); + // -listfileset + psMetadata *listfilesetsArgs = psMetadataAlloc(); + psMetadataAddS64(listfilesetsArgs, PS_LIST_TAIL, "-dist_id", 0, "list filesets with dist_id", 0); + psMetadataAddS64(listfilesetsArgs, PS_LIST_TAIL, "-target_id", 0, "list filesets with target_id", 0); + psMetadataAddS64(listfilesetsArgs, PS_LIST_TAIL, "-dest_id", 0, "list filesets with dest_id", 0); + psMetadataAddS64(listfilesetsArgs, PS_LIST_TAIL, "-int_id", 0, "list filesets with int_id", 0); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-dest_name", 0, "list filesets for destinationn name)", NULL); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-dist_group", 0, "list filesets for dist_group (LIKE comparison)", NULL); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-filter", 0, "list filesets by filter (LIKE comparison)", NULL); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-stage", 0, "list filesets for stage", NULL); + psMetadataAddBool(listfilesetsArgs, PS_LIST_TAIL,"-clean", 0, "list clean filesets", false); + psMetadataAddBool(listfilesetsArgs, PS_LIST_TAIL,"-full", 0, "list full filesets", false); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-state", 0, "list filesets in state", NULL); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); + psMetadataAddU64(listfilesetsArgs, PS_LIST_TAIL, "-limit", 0, "limit number of filesets listed to N", 0); + psMetadataAddBool(listfilesetsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -queuercrun psMetadata *queuercrunArgs = psMetadataAlloc(); @@ -228,15 +267,15 @@ psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); - // -listtarget - psMetadata *listtargetArgs = psMetadataAlloc(); - psMetadataAddS64(listtargetArgs, PS_LIST_TAIL, "-target_id", 0, "list target with target_id", 0); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-dist_group", 0, "list targets for dist_group", NULL); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-stage", 0, "list targets for stage", NULL); - psMetadataAddBool(listtargetArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); - psMetadataAddBool(listtargetArgs, PS_LIST_TAIL,"-full", 0, "list full targets", false); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-state", 0, "list targets in state", NULL); - psMetadataAddU64(listtargetArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); - psMetadataAddBool(listtargetArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -listtargets + psMetadata *listtargetsArgs = psMetadataAlloc(); + psMetadataAddS64(listtargetsArgs, PS_LIST_TAIL, "-target_id", 0, "list target with target_id", 0); + psMetadataAddStr(listtargetsArgs, PS_LIST_TAIL, "-dist_group", 0, "list targets for dist_group", NULL); + psMetadataAddStr(listtargetsArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); + psMetadataAddStr(listtargetsArgs, PS_LIST_TAIL, "-stage", 0, "list targets for stage", NULL); + psMetadataAddBool(listtargetsArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); + psMetadataAddBool(listtargetsArgs, PS_LIST_TAIL,"-full", 0, "list full targets", false); + psMetadataAddStr(listtargetsArgs, PS_LIST_TAIL, "-state", 0, "list targets in state", NULL); + psMetadataAddU64(listtargetsArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); + psMetadataAddBool(listtargetsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -defineinterest @@ -258,4 +297,10 @@ psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-dest_name", 0, "define destination name", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-dist_group", 0, "define distribution group", NULL); + psMetadataAddBool(updateinterestArgs, PS_LIST_TAIL,"-clean", 0, "update clean interests", false); + psMetadataAddBool(updateinterestArgs, PS_LIST_TAIL,"-full", 0, "update full interests", false); // -listinterests @@ -285,6 +330,9 @@ PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); PXOPT_ADD_MODE("-toadvance", "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs); + PXOPT_ADD_MODE("-pendingcleanup", "", DISTTOOL_MODE_PENDINGCLEANUP, pendingcleanupArgs); PXOPT_ADD_MODE("-pendingfileset", "", DISTTOOL_MODE_PENDINGFILESET, pendingfilesetArgs); PXOPT_ADD_MODE("-addfileset", "", DISTTOOL_MODE_ADDFILESET, addfilesetArgs); + PXOPT_ADD_MODE("-updatefileset", "", DISTTOOL_MODE_UPDATEFILESET, updatefilesetArgs); + PXOPT_ADD_MODE("-listfilesets", "", DISTTOOL_MODE_LISTFILESETS, listfilesetsArgs); PXOPT_ADD_MODE("-revertfileset", "", DISTTOOL_MODE_REVERTFILESET, revertfilesetArgs); PXOPT_ADD_MODE("-queuercrun", "", DISTTOOL_MODE_QUEUERCRUN, queuercrunArgs); @@ -299,5 +347,5 @@ PXOPT_ADD_MODE("-definetarget", "", DISTTOOL_MODE_DEFINETARGET, definetargetArgs); PXOPT_ADD_MODE("-updatetarget", "", DISTTOOL_MODE_UPDATETARGET, updatetargetArgs); - PXOPT_ADD_MODE("-listtarget", "", DISTTOOL_MODE_LISTTARGET, listtargetArgs); + PXOPT_ADD_MODE("-listtargets", "", DISTTOOL_MODE_LISTTARGETS, listtargetsArgs); PXOPT_ADD_MODE("-defineinterest", "", DISTTOOL_MODE_DEFINEINTEREST, defineinterestArgs); Index: branches/tap_branches/ippTools/src/dqstatstool.c =================================================================== --- branches/tap_branches/ippTools/src/dqstatstool.c (revision 27838) +++ branches/tap_branches/ippTools/src/dqstatstool.c (revision 27838) @@ -0,0 +1,799 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdqstats.h" +#include "dqstatstool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool createbundleMode(pxConfig *config); +//static bool cleanbundleMode(pxConfig *config); +static bool pendingbundleMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool revertrunMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = dqstatstoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DQSTATS_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DQSTATS_MODE_PENDINGBUNDLE, pendingbundleMode); + MODECASE(DQSTATS_MODE_CREATEBUNDLE, createbundleMode); + // MODECASE(DQSTATS_MODE_CLEANBUNDLE, cleanbundleMode); + MODECASE(DQSTATS_MODE_UPDATERUN, updaterunMode); + MODECASE(DQSTATS_MODE_REVERTRUN, revertrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + //psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(EXIT_SUCCESS); + + FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + // psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exit_status); +} + +static bool definebyqueryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,NULL); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(force, config->args, "-force", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, 0); + + // properties of the raw exposures + psMetadata *where = psMetadataAlloc(); + + pxdqstatsGetSearchArgs(config, where); + + psString query = pxDataGet("dqstatstool_definebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return(false); + } + + if (label) { + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + } + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (force) { + psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1 OR invalid = 0) "); + } + else { + psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1) "); + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + psTrace("debug",3,">> %s",query); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + // printf("%s\n",query); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dqstatstool", PS_LOG_INFO, "no rows found"); + psWarning("dqstatstool: no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "dqstatsContent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + // Create the run entry and grab the dqstats_id + dqstatsRunRow *run = dqstatsRunRowAlloc( + 0, // ID + "new", // state + registered, + set_label, + 0 // fault + ); + + if (!dqstatsRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + return(false); + } + + psS64 dqstats_id = psDBLastInsertID(config->dbh); + + psArray *list = psArrayAllocEmpty(16); // List of runs to print + psArrayAdd(list, list->n, run); + if (!dqstatsRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return(false); + } + psFree(list); + + // Create the content entry. + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + psS64 chip_id= psMetadataLookupS64(&status, row, "chip_id"); + psS64 cam_id = psMetadataLookupS64(&status, row, "cam_id"); + psS64 warp_id= psMetadataLookupS64(&status, row, "warp_id"); + + if (!dqstatsContentInsert(config->dbh, + dqstats_id, + exp_id, + chip_id, + cam_id, + warp_id, + 0 // do not create automatically invalid contents + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + return(true); +} + +static bool pendingbundleMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + + psMetadata *where = psMetadataAlloc(); + // pxdqstatsGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstatsRun.dqstats_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "dqstatsRun.label", "=="); + + psString query = pxDataGet("dqstatstool_get_run.sql"); // query + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retrieve sql statement (dqstatstool_get_run.sql)"); + return(false); + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); // where whereClause + + psArray *output = p_psDBFetchResult(config->dbh); // where whereClause output + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("dqstatstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "dqstatsPendingBundle", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return(true); +} +//static bool cleanbundleMode(pxConfig *config) { +// return(true); +//} + + +static bool createbundleMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + bool status = true; + PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", true, 0); + PXOPT_LOOKUP_STR(bundleUri, config->args, "-uri", true, 0); + + // Determine the information about this run. + psString runQuery = pxDataGet("dqstatstool_get_run.sql"); + if (!runQuery) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retrieve sql statement (dqstatstool_get_run.sql)"); + return(false); + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); // Save for content query + psStringAppend(&runQuery, " AND %s", whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, runQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(runQuery); + psFree(whereClause); + return(false); + } + psFree(runQuery); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + return(false); + } + + if (psArrayLength(output) != 1) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool: single return row not found."); + psFree(output); + psFree(whereClause); + return(false); + } + + psMetadata *run = output->data[0]; + if (strcmp(psMetadataLookupStr(&status,run,"state"),"new") != 0) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool: bundle already created for this run (should not happen)."); + psFree(output); + psFree(whereClause); + return(false); + } + psFree(output); // This held the information about the Run. + + // Read column list from config file + psMetadata *recipe = psMetadataLookupPtr(&status, config->modules->recipes, "DQSTATS"); + psMetadata *columns = psMetadataLookupMetadata(&status,recipe,"STATS.COLUMNS"); + if (!columns) { + psError(PS_ERR_UNKNOWN, false, "Could not read STATS.COLUMN metadata"); + psFree(whereClause); + return(false); + } + + psMetadataIterator *iter = psMetadataIteratorAlloc(columns, PS_LIST_HEAD,NULL); + + psMetadataItem *colItem = NULL; + psString columnlist = NULL; + + psMetadata *colNames = psMetadataAlloc(); + psMetadata *colTypes = psMetadataAlloc(); + while ((colItem = psMetadataGetAndIncrement(iter))) { + if (strcmp(colItem->name, "COLUMN")) continue; + if (colItem->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool COLUMN not of type METADATA\n"); + return(NULL); + } + + bool status; + psMetadata *coldef = colItem->data.md; + + if (columnlist) { // Stick intervening commas + psStringAppend(&columnlist, ", "); + } + + psStringAppend(&columnlist, "%s.%s", + psMetadataLookupStr(&status,coldef,"TABLE"), + psMetadataLookupStr(&status,coldef,"ENTRY")); + + psMetadataAddStr(colNames,PS_LIST_TAIL, + psMetadataLookupStr(&status,coldef,"ENTRY"),0,"", + psMetadataLookupStr(&status,coldef,"COLNAME")); + psMetadataAddStr(colTypes,PS_LIST_TAIL, + psMetadataLookupStr(&status,coldef,"ENTRY"),0,"", + psMetadataLookupStr(&status,coldef,"DATATYPE")); + } + psFree(iter); + + if (!columnlist) { + psError(PS_ERR_UNKNOWN, false, "Failed to find a columnlist"); + psFree(whereClause); + psFree(colNames); + psFree(colTypes); + return(false); + } + + // Read decision engine settings from config file. + psMetadata *engine = psMetadataLookupMetadata(&status,recipe,"DECISION.ENGINE"); + if (!engine) { + psError(PS_ERR_UNKNOWN, false, "Could not read DECISION.ENGINE metadata"); + psFree(whereClause); + return(false); + } + psArray *decisionRules = psArrayAllocEmpty(4); // list of screened and verified rules. + psMetadataIterator *iterEngine = psMetadataIteratorAlloc(engine, PS_LIST_HEAD,NULL); + psMetadataItem *engineItem = NULL; + + while ((engineItem = psMetadataGetAndIncrement(iterEngine))) { + if (strcmp(engineItem->name, "RULE")) continue; + if (engineItem->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool RULE not of type METADATA\n"); + return(NULL); + } + + bool status; + psMetadata *enginedef = engineItem->data.md; + + if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& + isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM"))) { + psArrayAdd(decisionRules,decisionRules->n,enginedef); + } + else { + psError(PS_ERR_UNKNOWN, false, "Rule invalid."); + } + } + psFree(iterEngine); + + if (decisionRules->n < 1) { + psError(PS_ERR_UNKNOWN, false, "Failed to generate a decision engine"); + psFree(decisionRules); + psFree(engine); + return(false); + } + psFree(engine); + + // Find the contents that comprise this run + psString contentQuery = pxDataGet("dqstatstool_get_contents.sql"); + if (!contentQuery) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retrieve sql statement (dqstatstool_get_contents.sql)"); + psFree(whereClause); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + return(false); + } + + psStringAppend(&contentQuery, " AND %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, contentQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contentQuery); + return(false); + } + psFree(contentQuery); + + psArray *contents = p_psDBFetchResult(config->dbh); + if (!contents) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + return(false); + } + if (!psArrayLength(contents)) { + psWarning("dqstatstool No rows found"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contents); + return(false); + } + + // Grab data from the database for a row of the bundle. + psString query = pxDataGet("dqstatstool_createbundle.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retrieve sql statement"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contents); + return(false); + } + // From disttool.c: + // "psStringSubstitute fails unless the input is a psString which it determines by + // comparing the memory blocks free function to an expected value. + // pxDataGet uses psSlurp which leaves a different free function on the memory block. + // To work around this make a copy of the query before doing the substitution." + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; + psStringSubstitute(&query,columnlist, "@COLLIST@"); + psFree(columnlist); + + psArray *outTable = psArrayAllocEmpty(contents->n); + for (long i = 0; i < contents->n; i++) { // Loop over content entries. + psMetadata *row = contents->data[i]; + + // Limit it just to the stuff we care about for this row. + bool status; + psString MYquery = psStringCopy(query); + if (!MYquery) { + psTrace("debug",4,"Failed to copy, trying again."); + MYquery = psStringCopy(query); + } + + psMetadata *where = psMetadataAlloc(); + + psMetadataAddS64(where,PS_LIST_TAIL, "rawExp.exp_id", 0, "==", psMetadataLookupS64(&status,row,"exp_id")); + psMetadataAddS64(where,PS_LIST_TAIL, "chipRun.chip_id", 0, "==", psMetadataLookupS64(&status,row,"chip_id")); + psMetadataAddS64(where,PS_LIST_TAIL, "camRun.cam_id", 0, "==", psMetadataLookupS64(&status,row,"cam_id")); + // This may not be real, so we need to check first. Is there a better way to check for a null here? + if (psMetadataLookupS64(&status,row,"warp_id") && (psMetadataLookupS64(&status,row,"warp_id") != PS_MAX_S64)) { + psMetadataAddS64(where,PS_LIST_TAIL, "warpRun.warp_id", 0, "==", psMetadataLookupS64(&status,row,"warp_id")); + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (!whereClause) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate Where Condition"); + return(false); + } + psStringAppend(&MYquery, " WHERE %s", whereClause); + +/* psTrace("debug",2,"%ld %ld %ld %ld %s\n",psMetadataLookupS64(&status,row,"exp_id"), */ +/* psMetadataLookupS64(&status,row,"chip_id"), */ +/* psMetadataLookupS64(&status,row,"cam_id"), */ +/* psMetadataLookupS64(&status,row,"warp_id"), */ +/* whereClause); */ + +/* psFree(whereClause); */ +/* psFree(where); */ + // printf("%s\n",MYquery); + + if (!p_psDBRunQuery(config->dbh, MYquery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(MYquery); + psFree(query); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(outTable); + return(false); + } + + psArray *statsoutput = p_psDBFetchResult(config->dbh); + if (!statsoutput) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + return(false); + } + if (!psArrayLength(statsoutput)) { + psWarning("dqstatstool No stats rows found: %ld %s %s",i,MYquery,query); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + return(false); + } + psFree(MYquery); + psFree(whereClause); + psFree(where); + + // Magic happens and the fits table is filled. + psMetadata *statsrow = statsoutput->data[0]; // Because there should only be one row. + + psMetadataIterator *statIter = psMetadataIteratorAlloc(statsrow,PS_LIST_HEAD,NULL); + + psMetadataItem *rowItem = NULL; + psMetadata *tableRow = psMetadataAlloc(); + // set args -dbname gpc1 -createbundle -dqstats_id 2 -uri /tmp/czwtest.fits + while ((rowItem = psMetadataGetAndIncrement(statIter))) { + bool success = false; + + psString colType = psMetadataLookupStr(&status,colTypes,rowItem->name); + if (!colType) { + psError(PS_ERR_UNKNOWN,false, "Could not find type for %s",rowItem->name); + return(false); + } + psString colName = psMetadataLookupStr(&status,colNames,rowItem->name); + if (!colName) { + psError(PS_ERR_UNKNOWN,false, "Could not find name for %s",rowItem->name); + return(false); + } + + if (!strcmp(colType,"PS_DATA_BOOL")) { + success = psMetadataAddBool(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.B); + } + else if (!strcmp(colType,"PS_DATA_S8")) { + success = psMetadataAddS8(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S8); + } + else if (!strcmp(colType,"PS_DATA_S16")) { + success = psMetadataAddS16(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S16); + } + else if (!strcmp(colType,"PS_DATA_S32")) { + success = psMetadataAddS32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S32); + } + else if (!strcmp(colType,"PS_DATA_S64")) { + success = psMetadataAddS64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S64); + } + else if (!strcmp(colType,"PS_DATA_U8")) { + success = psMetadataAddU8(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U8); + } + else if (!strcmp(colType,"PS_DATA_U16")) { + success = psMetadataAddU16(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U16); + } + else if (!strcmp(colType,"PS_DATA_U32")) { + success = psMetadataAddU32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U32); + } + else if (!strcmp(colType,"PS_DATA_U64")) { + success = psMetadataAddU64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U64); + } + else if (!strcmp(colType,"PS_DATA_F32")) { + success = psMetadataAddF32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.F32); + } + else if (!strcmp(colType,"PS_DATA_F64")) { + success = psMetadataAddF64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.F64); + } + else if (!strcmp(colType,"PS_DATA_STRING")) { + success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.str); + } + else if (!strcmp(colType,"PS_DATA_TIME")) { + psString timeString = psTimeToISO(rowItem->data.V); + success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",timeString); + psFree(timeString); + } + + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error constructing fits table row. %s %s",colType,colName); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + psFree(colType); + psFree(colName); + return(false); + } + } + // Calculate if this exposure was "good" or not. + bool accept = true; + for (long j = 0; j < decisionRules->n; j++) { + psMetadata *rule = decisionRules->data[j]; + bool status = true; + psString colname = psMetadataLookupStr(&status,rule,"COLNAME"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column name"); + return(false); + } + psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); + return(false); + } + psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); + return(false); + } + // Not happy with this being set to a F32. Can this ever be something else? + psF32 value = psMetadataLookupF32(&status,tableRow,colname); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f %f %f) in compare with rule.",colname,min_value,max_value,value); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(tableRow); + return(false); + } + if ((value > max_value)||(value < min_value)) { + accept = false; + } + } + bool success; + success = psMetadataAddBool(tableRow,PS_LIST_TAIL,"ACCEPT",0,"",accept); + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error adding exposure quality to fits table row."); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + return(false); + } + + // Add the row to the table array. + psArrayAdd(outTable,0,tableRow); + psFree(tableRow); + } + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + // Define the fits table here. + + psFits *Table = psFitsOpen(bundleUri,"w"); + if (!Table) { + psError(PS_ERR_UNKNOWN, false, "Could not open the fits table for writing."); + psFree(outTable); + return(false); + } + + psMetadata *header = psMetadataAlloc(); + psMetadataAddS64(header, PS_LIST_TAIL, "DQSTATS_ID", 0, "ID number of DQSTATS run", dqstats_id); + + if (!psFitsWriteTable(Table,header,outTable,"DQSTATS")) { + psError(PS_ERR_UNKNOWN, false, "Unable to write table to fits file."); + psFree(outTable); + psFree(Table); + psFree(header); + return(false); + } + psFree(outTable); + psFree(header); + + // Tear down, cleanup. + psFitsClose(Table); + return(true); +} + + +static bool updaterunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + if (state && ! pxIsValidState(state)) { + psError(PXTOOLS_ERR_CONFIG, false, "pxIsValidState failed"); + return(false); + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + PXOPT_COPY_S64(config->args, where, "-state", "state", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return(false); + } + + psString query = psStringCopy("UPDATE dqstatsRun "); + psStringAppend(&query," SET state = '%s' ", state); + if (fault) { + psStringAppend(&query, ", fault = %d ", fault); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query," WHERE %s ",whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + psFree(query); + psFree(where); + psFree(whereClause); + + return(true); +} + +static bool revertrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return(false); + } + + psString query = psStringCopy("UPDATE dqstatsRun SET fault = 0 "); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query," WHERE fault != 0 AND %s ",whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + psFree(query); + psFree(whereClause); + psFree(where); + + return(true); +} + + + + + + + Index: branches/tap_branches/ippTools/src/dqstatstool.h =================================================================== --- branches/tap_branches/ippTools/src/dqstatstool.h (revision 27838) +++ branches/tap_branches/ippTools/src/dqstatstool.h (revision 27838) @@ -0,0 +1,18 @@ +#ifndef DQSTATS_H +#define DQSTATS_H 1 + +#include "pxtools.h" + +typedef enum { + DQSTATS_MODE_NONE = 0x0, + DQSTATS_MODE_DEFINEBYQUERY, + DQSTATS_MODE_PENDINGBUNDLE, + DQSTATS_MODE_CREATEBUNDLE, + DQSTATS_MODE_CLEANBUNDLE, + DQSTATS_MODE_UPDATERUN, + DQSTATS_MODE_REVERTRUN, +} dqstatstoolMode; + +pxConfig *dqstatstoolConfig(pxConfig *config, int argc, char **argv); + +#endif Index: branches/tap_branches/ippTools/src/dqstatstoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/dqstatstoolConfig.c (revision 27838) +++ branches/tap_branches/ippTools/src/dqstatstoolConfig.c (revision 27838) @@ -0,0 +1,100 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include "pxtools.h" +#include "pxdqstats.h" +#include "dqstatstool.h" + +pxConfig *dqstatstoolConfig(pxConfig *config, int argc, char **argv) { + + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, "DQSTATS"); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return(NULL); + } + psTime *now = psTimeGetNow(PS_TIME_TAI); + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "set run label", NULL); + + pxdqstatsSetSearchArgs(definebyqueryArgs); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-force", 0, "invalidate old entries that match", false); + psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit bundle size to N rows", 0); + + // -createbundle + psMetadata *createbundleArgs = psMetadataAlloc(); + psMetadataAddS64(createbundleArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddStr(createbundleArgs, PS_LIST_TAIL, "-uri", 0, "output table URI", 0); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by dqstatsRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set dqstatsRun stats", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -pendingbundle + psMetadata *pendingbundleArgs = psMetadataAlloc(); + psMetadataAddS64(pendingbundleArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddU64(pendingbundleArgs, PS_LIST_TAIL, "-limit", 0, "limit number of bundles to N", 0); + psMetadataAddBool(pendingbundleArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(pendingbundleArgs, PS_LIST_TAIL, "-label", 0, "search by dqstatsRun.label", NULL); + + // -revertrun + psMetadata *revertrunArgs = psMetadataAlloc(); + psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label", 0, "search by dqstatsRun.label", NULL); + psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", DQSTATS_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change dqstats run properties", DQSTATS_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-createbundle", "create the bundle for a run", DQSTATS_MODE_CREATEBUNDLE, createbundleArgs); + PXOPT_ADD_MODE("-pendingbundle", "list bundles needing creation", DQSTATS_MODE_PENDINGBUNDLE, pendingbundleArgs); + PXOPT_ADD_MODE("-revertrun", "revert runs with errors", DQSTATS_MODE_REVERTRUN, revertrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return(NULL); + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return(NULL); + } + + // config = psMetadataConfigRead(config,1,"dqstatsTool.config",1); + + return config; +} + + + + Index: branches/tap_branches/ippTools/src/faketool.c =================================================================== --- branches/tap_branches/ippTools/src/faketool.c (revision 25900) +++ branches/tap_branches/ippTools/src/faketool.c (revision 27838) @@ -92,5 +92,5 @@ MODECASE(FAKETOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); - MODECASE(FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); + MODECASE(FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(FAKETOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(FAKETOOL_MODE_IMPORTRUN, importrunMode); @@ -123,4 +123,6 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); @@ -130,5 +132,5 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); // XXX does this choose the right label? PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); @@ -166,5 +168,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -188,5 +190,5 @@ psString query = pxDataGet("faketool_find_camrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -339,5 +341,5 @@ psFree(where); where = NULL; - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -346,7 +348,7 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "fakeRun", true); + bool result = pxUpdateRun(config, where, &query, "fakeRun", "fake_id", "fakeProcessedImfile", true); if (!result) { - psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + psError(psErrorCodeLast(), false, "pxUpdateRun failed"); } @@ -373,5 +375,5 @@ psString query = pxDataGet("faketool_find_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -440,5 +442,5 @@ psString query = pxDataGet("faketool_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -556,15 +558,15 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); psString query = pxDataGet("faketool_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "fakeProcessedImfile"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -665,5 +667,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -671,5 +673,5 @@ psString query = pxDataGet("faketool_revertprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -812,5 +814,5 @@ psString query = pxDataGet("faketool_unmasked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -889,5 +891,5 @@ psString query = pxDataGet("faketool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -954,5 +956,5 @@ psString query = pxDataGet("faketool_pendingcleanupimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1015,5 +1017,5 @@ psString query = pxDataGet("faketool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1077,5 +1079,5 @@ psString query = pxDataGet("faketool_completely_processed_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1285,5 +1287,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/faketoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/faketoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/faketoolConfig.c (revision 27838) @@ -39,5 +39,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -47,4 +47,6 @@ psMetadata *queueArgs = psMetadataAlloc(); // XXX need to allow multiple exp_ids + psMetadataAddS64(queueArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(queueArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); psMetadataAddS64(queueArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); Index: branches/tap_branches/ippTools/src/flatcorr.c =================================================================== --- branches/tap_branches/ippTools/src/flatcorr.c (revision 25900) +++ branches/tap_branches/ippTools/src/flatcorr.c (revision 27838) @@ -115,9 +115,9 @@ if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - // require the camera to be defined: this analysis does not make sense + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // require the camera to be defined: this analysis does not make sense // across multiple cameras PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); @@ -133,4 +133,5 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + PXOPT_LOOKUP_BOOL(make_correction, config->args, "-make_correction", false); PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); @@ -141,5 +142,5 @@ psString query = pxDataGet("chiptool_find_rawexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -172,12 +173,12 @@ if (pretend) { - for (long i = 0; i < psArrayLength(output); i++) { - // negative simple so the default is true - if (!ippdbPrintMetadataRaw(stdout, output->data[i], !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } + for (long i = 0; i < psArrayLength(output); i++) { + // negative simple so the default is true + if (!ippdbPrintMetadataRaw(stdout, output->data[i], !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } psFree(output); return true; @@ -194,19 +195,20 @@ // create a new flatcorrRun if (!flatcorrRunInsert( - config->dbh, + config->dbh, 0, // corr_id - det_type, + det_type, dvodb, - camera, - telescope, - NULL, + camera, + telescope, + NULL, filter, "reg", // state + make_correction, workdir, label, reduction, - region, - NULL, - 0 + region, + NULL, + 0 )) { if (!psDBRollback(config->dbh)) { @@ -294,4 +296,5 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + PXOPT_LOOKUP_BOOL(make_correction, config->args, "-make_correction", false); // XXX probably should make the region in -set_region match ra_min, ra_max, etc @@ -307,18 +310,19 @@ // create a new flatcorrRun flatcorrRunRow *row = flatcorrRunRowAlloc( - 0, // corr_id - det_type, - dvodb, - camera, - telescope, - NULL, - filter, - "reg", // state - workdir, - label, - reduction, - region, - NULL, // hostname - 0 // fault + 0, // corr_id + det_type, + dvodb, + camera, + telescope, + NULL, + filter, + "reg", // state + make_correction, + workdir, + label, + reduction, + region, + NULL, // hostname + 0 // fault ); @@ -331,5 +335,5 @@ return false; } - + // figure out the ID of the flatcorrRun we just created psS64 corr_id = psDBLastInsertID(config->dbh); @@ -373,6 +377,6 @@ psString query = pxDataGet("flatcorr_dropchip.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; } @@ -416,6 +420,6 @@ psString query = pxDataGet("flatcorr_dropcamera.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; } @@ -444,6 +448,6 @@ psString query = pxDataGet("flatcorr_chiprundone.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; } @@ -457,7 +461,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -520,15 +524,16 @@ // queue the exp if (!pxcamQueueByChipID( - config, - row->chip_id, - row->workdir, - row->label, - row->data_group, - row->dist_group, - row->reduction, - row->expgroup, - row->dvodb, - row->tess_id, - "camera", + config, + row->chip_id, + row->workdir, + row->label, + row->data_group, + row->dist_group, + row->reduction, + row->expgroup, + row->dvodb, + row->tess_id, + "camera", + row->magicked, NULL // note does not propragate )) { @@ -543,8 +548,8 @@ } - // figure out the ID of the flatcorrRun we just created - psS64 cam_id = psDBLastInsertID(config->dbh); - - // add the camRun entry to the flatcorrCamLink table (include is TRUE) + // figure out the ID of the flatcorrRun we just created + psS64 cam_id = psDBLastInsertID(config->dbh); + + // add the camRun entry to the flatcorrCamLink table (include is TRUE) if (!flatcorrCamLinkInsert(config->dbh, corr_id, row->chip_id, cam_id, 1)) { if (!psDBRollback(config->dbh)) { @@ -554,5 +559,5 @@ return false; } - + psFree(row); } @@ -582,6 +587,6 @@ psString query = pxDataGet("flatcorr_camerarundone.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; } @@ -595,7 +600,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -658,13 +663,13 @@ // queue the exp : force image_only to be false (flatcorr is meaningless with just image info) if (!pxaddQueueByCamID( - config, - row->cam_id, - row->workdir, - row->reduction, - row->label, + config, + row->cam_id, + row->workdir, + row->reduction, + row->label, row->data_group, - row->dvodb, + row->dvodb, NULL, // note is not propagated - 0)) { + 0)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -677,8 +682,8 @@ } - // figure out the ID of the flatcorrRun we just created - psS64 add_id = psDBLastInsertID(config->dbh); - - // add the addRun entry to the flatcorrAddstarLink table (include is TRUE) + // figure out the ID of the flatcorrRun we just created + psS64 add_id = psDBLastInsertID(config->dbh); + + // add the addRun entry to the flatcorrAddstarLink table (include is TRUE) if (!flatcorrAddstarLinkInsert(config->dbh, corr_id, row->cam_id, add_id, 1)) { if (!psDBRollback(config->dbh)) { @@ -714,6 +719,6 @@ psString query = pxDataGet("flatcorr_pendingprocess.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; } @@ -727,7 +732,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -750,7 +755,7 @@ if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; } @@ -809,7 +814,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -832,7 +837,7 @@ if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; } @@ -849,5 +854,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); - + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "=="); @@ -855,5 +860,5 @@ psString query = pxDataGet("flatcorr_inputimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -868,7 +873,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -891,7 +896,7 @@ if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; } @@ -905,7 +910,7 @@ // check that state is a valid string value - if (!strcmp(state, "reg") && - !strcmp(state, "new") && - !strcmp(state, "full")) + if (!strcmp(state, "reg") && + !strcmp(state, "new") && + !strcmp(state, "full")) { psError(PS_ERR_UNKNOWN, false, "invalid state: %s", state); Index: branches/tap_branches/ippTools/src/flatcorrConfig.c =================================================================== --- branches/tap_branches/ippTools/src/flatcorrConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/flatcorrConfig.c (revision 27838) @@ -59,4 +59,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessalation", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_region", 0, "define region", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-make_correction", 0, "generate a correction image and add to detrend database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); @@ -76,4 +77,5 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessalation", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_region", 0, "define region", NULL); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-make_correction", 0, "generate a correction image and add to detrend database", false); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -129,4 +131,5 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-corr_id", 0, "define correction id (required)", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + // XXX add mechanism to change this value: psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-set_correction", 0, "generate a correction image and add to detrend database", false); // -inputexp Index: branches/tap_branches/ippTools/src/magicdstool.c =================================================================== --- branches/tap_branches/ippTools/src/magicdstool.c (revision 25900) +++ branches/tap_branches/ippTools/src/magicdstool.c (revision 27838) @@ -33,4 +33,5 @@ static bool definebyqueryMode(pxConfig *config); +static bool definecopyMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool todestreakMode(pxConfig *config); @@ -38,4 +39,5 @@ static bool advancerunMode(pxConfig *config); static bool revertdestreakedfileMode(pxConfig *config); +static bool clearstatefaultsMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); static bool toremoveMode(pxConfig *config); @@ -44,5 +46,5 @@ static bool tocleanupMode(pxConfig *config); -static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state); +static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state); # define MODECASE(caseName, func) \ @@ -65,4 +67,5 @@ switch (config->mode) { MODECASE(MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MAGICDSTOOL_MODE_DEFINECOPY, definecopyMode); MODECASE(MAGICDSTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(MAGICDSTOOL_MODE_TODESTREAK, todestreakMode); @@ -70,4 +73,5 @@ MODECASE(MAGICDSTOOL_MODE_ADVANCERUN, advancerunMode); MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); + MODECASE(MAGICDSTOOL_MODE_CLEARSTATEFAULTS, clearstatefaultsMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); MODECASE(MAGICDSTOOL_MODE_TOREMOVE, toremoveMode); @@ -110,5 +114,5 @@ PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); - PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -124,8 +128,8 @@ PXOPT_COPY_S32(config->args, where, "-streaks_max","streaks", "<="); - pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); // define using magicRun label ippStage stageNum = ippStringToStage(stage); - + psString query = NULL; switch (stageNum) { @@ -147,8 +151,8 @@ case IPP_STAGE_FAKE: case IPP_STAGE_STACK: - psError(PXTOOLS_ERR_DATA, true, "%sRuns do not need to be destreaked", stage); + psError(PXTOOLS_ERR_CONFIG, true, "%sRuns do not need to be destreaked", stage); return false; case IPP_STAGE_NONE: - psError(PXTOOLS_ERR_DATA, true, "%s is not a valid stage", stage); + psError(PXTOOLS_ERR_CONFIG, true, "%s is not a valid stage", stage); return false; default: @@ -159,5 +163,5 @@ if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -233,5 +237,5 @@ // Parse the list of runs ready to be destreaked - if (!dry_run && !psDBTransaction(config->dbh)) { + if (!pretend && !psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -249,5 +253,5 @@ psString magicRunDataGroup = psMetadataLookupStr(NULL, row, "data_group"); psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir"); - + // if workdir is not supplied use the magicRun's if (!workdir) { @@ -282,5 +286,5 @@ } - if (!dry_run) { + if (!pretend) { if (!magicDSRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -301,5 +305,5 @@ } - if (!dry_run && !psDBCommit(config->dbh)) { + if (!pretend && !psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -319,4 +323,224 @@ +// XXX This currently allows multiple destreak runs to be queued on the same exposure if there are multiple +// magicRuns selected! +static bool definecopyMode(pxConfig *config) +{ + // Required + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // Optional + PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // search args + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "chipRun.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_S32(config->args, where, "-streaks_max", "magicMask.streaks", "<="); + + pxAddLabelSearchArgs (config, where, "-magic_label", "magicRun.label", "=="); // define magic label + psString labelName = NULL; // Name of label + psStringAppend(&labelName, "%sRun.label", stage); + pxAddLabelSearchArgs (config, where, "-stage_label", labelName, "=="); // define stageRun label + psFree(labelName); + + ippStage stageNum = ippStringToStage(stage); + + psString query = NULL; + switch (stageNum) { + case IPP_STAGE_RAW: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Raw stage is not appropriate for a copied destreak"); + return false; + case IPP_STAGE_CHIP: + query = pxDataGet("magicdstool_definecopy_chip.sql"); + break; + case IPP_STAGE_WARP: + query = pxDataGet("magicdstool_definecopy_warp.sql"); + break; + case IPP_STAGE_CAMERA: + case IPP_STAGE_DIFF: + case IPP_STAGE_FAKE: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "%s has not been coded.", stage); + return false; + case IPP_STAGE_STACK: + psError(PXTOOLS_ERR_CONFIG, true, "Stacks do not need to be destreaked"); + return false; + case IPP_STAGE_NONE: + psError(PXTOOLS_ERR_CONFIG, true, "%s is not a valid stage", stage); + return false; + default: + psError(PXTOOLS_ERR_PROG, true, "ippStageToString returned %d for invalid stage %s", + stageNum, stage); + return false; + } + + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + const char *rerun_flag = rerun ? "" : "\n"; // String to give query to activate (or not) rerun + + if (stageNum != IPP_STAGE_DIFF) { + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQueryF(config->dbh, query, set_label, rerun_flag)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + } else { + // diff stage query has two types bothways and !bothways + // so we need to send the rerun flag and the where data twice + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, rerun_flag, whereString, rerun_flag, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + psFree(query); + return false; + } + psFree(whereString); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // Parse the list of runs ready to be destreaked + + if (!pretend && !psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; // Row of interest + psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); + psS64 inv_magic_id = psMetadataLookupS64(NULL, row, "inv_magic_id"); + psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); + psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunDataGroup = psMetadataLookupStr(NULL, row, "data_group"); + + psString outroot = NULL; + // set outroot to workdir/exp_id/stage for example /somewhere/424242/chip + psStringAppend(&outroot, "%s/%" PRId64 "/%s", workdir, exp_id, stage); + + // create a new magicRun for this group + magicDSRunRow *run = magicDSRunRowAlloc( + 0, // magic_ds_id + magic_id, + inv_magic_id, + "new", + stage, + stage_id, + cam_id, + set_label ? set_label : magicRunLabel, + set_data_group ? set_data_group : magicRunDataGroup, + outroot, + recoveryroot, + noreplace ? 0 :1, // re_place + 0, // remove + 0, // fault + note); // remove + + psFree(outroot); + if (!run) { + psAbort("failed to alloc magicDSRun object"); + } + + if (!pretend) { + if (!magicDSRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psS64 magic_ds_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_ds_id = magic_ds_id; + } + + psArrayAdd(list, list->n, run); + psFree(run); + } + + if (!pretend && !psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); + + if (!magicDSRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(list); + + return true; +} + + static bool updaterunMode(pxConfig *config) { @@ -326,8 +550,19 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + // optional + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + psString setString = NULL; + if (set_label) { + psStringAppend(&setString, ", label = '%s'", set_label); + } + if (set_data_group) { + psStringAppend(&setString, ", data_group = '%s'", set_data_group); + } + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); if (magic_ds_id) { - return setmagicDSRunState(config, magic_ds_id, NULL, state); + return setmagicDSRunState(config, magic_ds_id, setString, NULL, state); } else if (!strcmp(state, "full")) { @@ -341,5 +576,6 @@ PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "LIKE"); if (psListLength(where->list) < 2) { @@ -353,5 +589,5 @@ psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0); } - bool result = setmagicDSRunState(config, magic_ds_id, where, state); + bool result = setmagicDSRunState(config, magic_ds_id, setString, where, state); psFree(where); @@ -379,5 +615,5 @@ psString query = pxDataGet(sql_file); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement from %s", sql_file); psFree(sql_file); return false; @@ -445,5 +681,5 @@ { // first query the magicDSRun to find the stage and the stage_id - psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64; + psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { @@ -472,4 +708,5 @@ psString stage= psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); ippStage stageNum = ippStringToStage(stage); @@ -499,5 +736,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -518,5 +755,5 @@ { // first query the magicDSRun to find the stage and the stage_id - psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64; + psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { @@ -545,4 +782,5 @@ psString stage = psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); ippStage stageNum = ippStringToStage(stage); @@ -570,5 +808,5 @@ return false; } - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -598,4 +836,7 @@ PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false); PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false); + PXOPT_LOOKUP_F32(streak_frac, config->args, "-streak_frac", false, false); + PXOPT_LOOKUP_F32(nondiff_frac, config->args, "-nondiff_frac", false, false); + PXOPT_LOOKUP_F32(run_time, config->args, "-run_time", false, false); if (setmagicked && (fault != 0)) { @@ -620,5 +861,15 @@ } - if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault, "full")) { + if (!magicDSFileInsert(config->dbh, + magic_ds_id, + component, + backup_path_base, + recovery_path_base, + streak_frac, + nondiff_frac, + run_time, + fault, + "full" // data_state + )) { // rollback if (!psDBRollback(config->dbh)) { @@ -651,5 +902,5 @@ psString query = pxDataGet("magicdstool_completed_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -699,5 +950,5 @@ // set magicDSRun.state to 'full' - if (!setmagicDSRunState(config, magic_ds_id, NULL, "full")) { + if (!setmagicDSRunState(config, magic_ds_id, NULL, NULL, "full")) { psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64, magic_ds_id); @@ -747,4 +998,49 @@ if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + psFree(query); + return false; + } + psFree(query); + return true; +} +static bool clearstatefaultsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + // new state + PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false); + // old state (required) + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("magicdstool_clearstatefaults.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; + } + psFree(where); + + if (!new_state) { + if (!strcmp(state, "failed_revert")) { + new_state = "new"; + } else if (!strcmp(state, "failed_cleanup")) { + new_state = "goto_cleaned"; + } else { + psError(PS_ERR_UNKNOWN, true, "unexpected value for state: %s", state); + return false; + } + } + if (!p_psDBRunQueryF(config->dbh, query, new_state)) { psError(PS_ERR_UNKNOWN, false, "failed to revert"); psFree(query); @@ -809,6 +1105,6 @@ for (int i=0; idata[i]; - psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); - psString old_state = psMetadataLookupStr(NULL, row, "state"); + psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); + psString old_state = psMetadataLookupStr(NULL, row, "state"); psString new_state; if (!strcmp(old_state, "goto_censored")) { @@ -838,4 +1134,5 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); @@ -845,16 +1142,16 @@ psString query = pxDataGet("magicdstool_getskycells.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause, "\n AND "); } psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -901,4 +1198,7 @@ if (!((strcmp(state, "new") == 0) || (strcmp(state, "full") == 0) || + (strcmp(state, "drop") == 0) || + (strcmp(state, "failed_revert") == 0) || + (strcmp(state, "failed_cleanup") == 0) || (strcmp(state, "restored") == 0) || (strcmp(state, "censored") == 0) || @@ -914,5 +1214,5 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psMetadata *where, const char *state) +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psString extraSetStr, psMetadata *where, const char *state) { PS_ASSERT_PTR_NON_NULL(state, false); @@ -924,5 +1224,6 @@ } - psString query = psStringCopy("UPDATE magicDSRun SET state = '%s'"); + psString query = NULL; + psStringAppend(&query, "UPDATE magicDSRun SET state = '%s' %s\n", state, extraSetStr ? extraSetStr : ""); if (magic_ds_id) { psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id); @@ -932,9 +1233,17 @@ psFree(whereClause); } else { - psError(PS_ERR_UNKNOWN, true, "search arugments are required"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; + } + + if (!strcmp(state, "goto_cleaned")) { + // Don't set state back to goto_cleaned if it is already cleaned + psStringAppend(&query, " AND (magicDSRun.state != 'cleaned')"); + + // don't clean up magicDSRun's where stage is camera + psStringAppend(&query, " AND (magicDSRun.stage != 'camera')"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for magic_id %" PRId64, magic_ds_id); @@ -960,5 +1269,5 @@ psString query = pxDataGet("magicdstool_toremove.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1038,5 +1347,5 @@ psString query = pxDataGet(sql_file); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement from %s", sql_file); psFree(sql_file); return false; @@ -1106,5 +1415,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); @@ -1116,5 +1425,5 @@ psString query = pxDataGet("magicdstool_tocleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/magicdstool.h =================================================================== --- branches/tap_branches/ippTools/src/magicdstool.h (revision 25900) +++ branches/tap_branches/ippTools/src/magicdstool.h (revision 27838) @@ -26,4 +26,5 @@ MAGICDSTOOL_MODE_NONE = 0x0, MAGICDSTOOL_MODE_DEFINEBYQUERY, + MAGICDSTOOL_MODE_DEFINECOPY, MAGICDSTOOL_MODE_UPDATERUN, MAGICDSTOOL_MODE_TODESTREAK, @@ -31,4 +32,5 @@ MAGICDSTOOL_MODE_ADVANCERUN, MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, + MAGICDSTOOL_MODE_CLEARSTATEFAULTS, MAGICDSTOOL_MODE_GETSKYCELLS, MAGICDSTOOL_MODE_TOREMOVE, Index: branches/tap_branches/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/magicdstoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/magicdstoolConfig.c (revision 27838) @@ -40,5 +40,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -66,16 +66,43 @@ psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-streaks_max", 0, "maximum number of streaks", 0); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one arleady exists", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be queued", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one already exists", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be queued", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -definecopy + psMetadata *definecopyArgs = psMetadataAlloc(); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-stage_label", PS_META_DUPLICATE_OK, "search by stageRun.label", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-magic_label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic_id", 0); + psMetadataAddS32(definecopyArgs, PS_LIST_TAIL, "-streaks_max", 0, "maximum number of streaks", 0); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one already exists", false); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be queued", false); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(definecopyArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "set data_group", NULL); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "data_group (LIKE) comparision", NULL); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-noreplace", 0, "only update runs with replace not set", false); @@ -102,4 +129,7 @@ psMetadataAddStr(adddestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL); psMetadataAddBool(adddestreakedfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false); + psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-streak_frac", 0, "set fraction of pixels masked by streaks", 0); + psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-nondiff_frac", 0, "set fraction of pixels masked because nondiffed", 0); + psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-run_time", 0, "set the streaksremove run time for component ", 0); psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -111,4 +141,12 @@ psMetadataAddBool(revertdestreakedfileArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirm that you know what you are doing", false); psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + + // -clearstatefaults + psMetadata *clearstatefaultsArgs = psMetadataAlloc(); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); + psMetadataAddS16(clearstatefaultsArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); // -getskycells @@ -163,4 +201,6 @@ PXOPT_ADD_MODE("-definebyquery", "create magic de-streak runs from magic runs", MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definecopy", "create magic de-streak runs from magic runs for different path", + MAGICDSTOOL_MODE_DEFINECOPY, definecopyArgs); PXOPT_ADD_MODE("-updaterun", "update state of magic de-streak run", MAGICDSTOOL_MODE_UPDATERUN, updaterunArgs); @@ -173,4 +213,6 @@ PXOPT_ADD_MODE("-revertdestreakedfile", " revert a faulted de-streaked file", MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, revertdestreakedfileArgs); + PXOPT_ADD_MODE("-clearstatefaults", " clear magicDSRuns in fault state", + MAGICDSTOOL_MODE_CLEARSTATEFAULTS, clearstatefaultsArgs); PXOPT_ADD_MODE("-getskycells", "get skycell files ", MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsArgs); Index: branches/tap_branches/ippTools/src/magictool.c =================================================================== --- branches/tap_branches/ippTools/src/magictool.c (revision 25900) +++ branches/tap_branches/ippTools/src/magictool.c (revision 27838) @@ -114,7 +114,7 @@ // Required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); // Optional - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); PXOPT_LOOKUP_STR(note, config->args, "-note", false, false); @@ -135,5 +135,5 @@ psString query = pxDataGet("magictool_definebyquery_select.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -151,6 +151,8 @@ // what if no skycells for the diff run completed? - if (!rerun) { - psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE"); + psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE"); + psString rerunWhereStr = NULL; + if (rerun) { + psStringAppend(&rerunWhereStr, "\n WHERE magicRun.label = '%s'", label); } @@ -176,9 +178,13 @@ queryWhereStr = psStringCopy(""); } - - if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) { + if (!rerunWhereStr) { + rerunWhereStr = psStringCopy(""); + } + + if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, rerunWhereStr, queryWhereStr)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(diffWhereStr); psFree(queryWhereStr); + psFree(rerunWhereStr); psFree(query); return false; @@ -186,4 +192,5 @@ psFree(diffWhereStr); psFree(queryWhereStr); + psFree(rerunWhereStr); psFree(query); } @@ -427,5 +434,5 @@ psString query = pxDataGet("magictool_inputskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -500,5 +507,5 @@ psString query = pxDataGet("magictool_totree.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -573,5 +580,5 @@ if (fault > 0) { - char *query = "UPDATE magicRun SET fault = %d, state = 'full' WHERE magic_id = %" PRId64; + char *query = "UPDATE magicRun SET fault = %d WHERE magic_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, fault, magic_id)) { psError(PS_ERR_UNKNOWN, false, @@ -612,4 +619,7 @@ return false; } + psS32 numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("magictool", PS_LOG_INFO, "Reverted %d magic runs", numUpdated); + return true; } @@ -635,5 +645,5 @@ psString query = pxDataGet("magictool_inputs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -805,5 +815,5 @@ psString query = pxDataGet("magictool_toprocess_inputs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -867,12 +877,21 @@ // look for tree nodes that need to be processed + + // first find incomplete magicRuns query = pxDataGet("magictool_toprocess_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // we limit the query even though it is cheap (only magic_id is selected) + + // XXX: if the first 1000 unfinished magicRuns have no ready nodes + // that haven't faulted, later runs won't get returned even though + // they have work to do. When we used a limit of 100 we actually ran + // into this problem. Since we're using labels a limit of 1000 will + // probably be ok. { - psString limitString = psDBGenerateLimitSQL( 100 ); + psString limitString = psDBGenerateLimitSQL( 1000 ); psStringAppend(&query, " %s", limitString); psFree(limitString); @@ -910,5 +929,5 @@ query = pxDataGet("magictool_toprocess_tree.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -923,5 +942,5 @@ psAbort("failed to lookup value for magic_id column"); } - + whereString = NULL; psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id); @@ -1041,10 +1060,10 @@ PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "magicNodeResult.fault", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); psString query = pxDataGet("magictool_revertnode.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1067,5 +1086,5 @@ psS32 numUpdated = psDBAffectedRows(config->dbh); - psLogMsg("magictool", PS_LOG_INFO, "Updated %d magic nodes", numUpdated); + psLogMsg("magictool", PS_LOG_INFO, "Reverted %d magic nodes", numUpdated); return true; @@ -1083,5 +1102,5 @@ psString query = pxDataGet("magictool_tomask.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1171,5 +1190,5 @@ psString query = pxDataGet("magictool_addmask.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1278,5 +1297,5 @@ psString query = pxDataGet("magictool_mask.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/magictoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/magictoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/magictoolConfig.c (revision 27838) @@ -40,5 +40,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -50,7 +50,9 @@ psMetadata *definebyqueryArgs = psMetadataAlloc(); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label (required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-note", 0, "define note", NULL); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search diff_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); @@ -135,5 +137,5 @@ psMetadataAddS64(revertnodeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-node", 0, "search by node name", NULL); - psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-label",0, "search by label", NULL); + psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); psMetadataAddS16(revertnodeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); Index: branches/tap_branches/ippTools/src/pstamptool.c =================================================================== --- branches/tap_branches/ippTools/src/pstamptool.c (revision 25900) +++ branches/tap_branches/ippTools/src/pstamptool.c (revision 27838) @@ -51,4 +51,5 @@ static bool pendingdependentMode(pxConfig *config); static bool updatedependentMode(pxConfig *config); +static bool revertdependentMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -91,4 +92,5 @@ MODECASE(PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentMode); MODECASE(PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentMode); + MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode); default: psAbort("invalid option (this should not happen)"); @@ -116,8 +118,10 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - PXOPT_LOOKUP_STR(outProduct, config->args, "-out_product", true, false); - PXOPT_LOOKUP_STR(lastFileset, config->args, "-last_fileset", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-set_uri", true, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-set_out_product", true, false); + PXOPT_LOOKUP_STR(lastFileset, config->args, "-set_last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval",false, false); if (!pstampDataStoreInsert(config->dbh, @@ -125,6 +129,9 @@ state, lastFileset, + NULL, // timestamp + label, // label outProduct, - uri + uri, + pollInterval )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -142,9 +149,10 @@ PXOPT_COPY_S64(config->args, where, "-ds_id", "ds_id", "=="); + PXOPT_LOOKUP_BOOL(ready, config->args, "-ready", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); psString query = pxDataGet("pstamptool_datastore.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -156,4 +164,7 @@ } psFree(where); + if (ready) { + psStringAppend(&query, " %s", "\nAND TIMESTAMPDIFF(SECOND, timestamp, now()) > pollInterval"); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -189,25 +200,33 @@ PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", true, false); - PXOPT_LOOKUP_STR(lastFileset, config->args, "-last_fileset", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); - - if (!state && !lastFileset) { - psError(PS_ERR_UNKNOWN, true, "at least one of -last_fileset or -state is required"); - return false; - } - - char *query = psStringCopy ("UPDATE pstampDataStore SET"); - bool needComma = false; - + PXOPT_LOOKUP_STR(lastFileset, config->args, "-set_last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval", false, false); + PXOPT_LOOKUP_BOOL(update_timestamp, config->args, "-update_timestamp", false); + + if (!state && !lastFileset && !pollInterval && !update_timestamp && !label) { + psError(PS_ERR_UNKNOWN, true, "at least one of -last_fileset or -set_state is required"); + return false; + } + + char *query = psStringCopy ("UPDATE pstampDataStore SET timestamp = CURRENT_TIMESTAMP() "); + if (lastFileset) { - psStringAppend(&query, " lastFileset = '%s'", lastFileset); - needComma = true; + psStringAppend(&query, " , lastFileset = '%s'", lastFileset); } if (state) { - psStringAppend(&query, "%s state = '%s'", needComma ? "," : " ", state); - needComma = true; // be ready in case we add another field - } - + psStringAppend(&query, ", state = '%s'", state); + } + + if (label) { + psStringAppend(&query, ", label = '%s'", label); + } + + if (pollInterval) { + psStringAppend(&query, ", pollInterval = %d", pollInterval); + } + psStringAppend(&query, " WHERE ds_id = %" PRId64, ds_id); @@ -220,5 +239,5 @@ psU64 affected = psDBAffectedRows(config->dbh); if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); return false; @@ -245,6 +264,6 @@ label, NULL, // outProduct - uri, - 0 // fault + uri, + 0 // fault )) { psError(PS_ERR_UNKNOWN, false, "failed to insert request"); @@ -265,4 +284,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -271,5 +291,5 @@ psString query = pxDataGet("pstamptool_pendingreq.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -342,5 +362,5 @@ psFree(whereClause); psFree(where); - + // treat limit == 0 as "no limit" if (limit) { @@ -365,5 +385,5 @@ psTrace("pstamptool", PS_LOG_INFO, "request not found"); // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for - psError(PXTOOLS_ERR_DATA, true, "request not found"); + psError(PXTOOLS_ERR_CONFIG, true, "request not found"); psFree(output); // we return false so that the caller can determine that a request does not exist @@ -387,8 +407,18 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); psString query = pxDataGet("pstamptool_completedreq.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" @@ -436,7 +466,7 @@ PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(outProduct, config->args, "-outProduct", false, false); - PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); PXOPT_LOOKUP_STR(reqType, config->args, "-reqType", false, false); @@ -454,5 +484,5 @@ } if (fault) { - psStringAppend(&query, "%c fault = %s", c, fault); + psStringAppend(&query, "%c fault = %d", c, fault); c = ','; } @@ -485,5 +515,5 @@ // note zero is not an error if (affected > 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); return false; @@ -500,10 +530,11 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -525,5 +556,5 @@ query = pxDataGet("pstamptool_revertreq.sql"); psStringAppend(&query, " AND %s", whereClause); - + psFree(whereClause); @@ -555,25 +586,25 @@ // unless the job is being inserted with stop state require outputBase if (strcmp(stateString, "stop") && !outputBase) { - psError(PS_ERR_UNKNOWN, true, "-outputBase is required"); - return false; + psError(PS_ERR_UNKNOWN, true, "-outputBase is required"); + return false; } // default value for job_type is defined in pstamptoolConfig.c if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query") || !strcmp(job_type, "none")) { - stampJob = false; + stampJob = false; } else if (!strcmp(job_type, "stamp")) { - stampJob = true; + stampJob = true; } else { - psError(PS_ERR_UNKNOWN, false, "unknown value for -job_type: %s", job_type); - return false; - } - if (!pstampJobInsert(config->dbh, + psError(PS_ERR_UNKNOWN, false, "unknown value for -job_type: %s", job_type); + return false; + } + if (!pstampJobInsert(config->dbh, 0, // job_id req_id, - rownum, - stateString, - job_type, - fault, - exp_id, + rownum, + stateString, + job_type, + fault, + exp_id, outputBase, options, @@ -586,5 +617,5 @@ psU64 affected = psDBAffectedRows(config->dbh); if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); @@ -617,5 +648,5 @@ psString query = pxDataGet("pstamptool_listjob.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -674,4 +705,6 @@ PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -679,5 +712,5 @@ psString query = pxDataGet("pstamptool_pendingjob.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -685,5 +718,5 @@ // use psDBGenerateWhereSQL because the SQL yields an intermediate table if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -732,7 +765,18 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false); + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); + + if (!job_id && !dep_id) { + psError(PS_ERR_UNKNOWN, true, "at least -job_id or -dep_id is required"); + return false; + } PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); psString faultStr = NULL; @@ -740,12 +784,15 @@ faultStr = psStringCopy(""); } else { - psStringAppend(&faultStr, ", fault = '%s'", fault); - } - - char *query ="UPDATE pstampJob" - " SET state = '%s' %s" - " WHERE job_id = %" PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, faultStr, job_id)) { + psStringAppend(&faultStr, "\n, pstampJob.fault = %d", fault); + } + + psString query = pxDataGet("pstamptool_updatejob.sql"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, state, faultStr)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -753,11 +800,8 @@ } psFree(faultStr); + psFree(query); psU64 affected = psDBAffectedRows(config->dbh); - if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" - PRIu64 " rows were modified", affected); - return false; - } + psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampJobs", affected); return true; @@ -772,23 +816,34 @@ PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "pstampJob.fault", "=="); PXOPT_COPY_S64(config->args, where, "-req_id_min", "req_id", ">="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + // By default only revert faults < 10 which are our "ipp exit codes" + // codes larger than that are the pstamp request interface. + // Don't fault those unless -fault waa provided + psString faultClause = ""; + if (!fault) { + faultClause = " \nAND (pstampJob.fault < 10)"; + } psString query = pxDataGet("pstamptool_revertjob.sql"); if (!psListLength(where->list) && !all) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters or -all are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters or -all are required"); return false; } if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { + + if (!p_psDBRunQueryF(config->dbh, query, faultClause)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -837,5 +892,5 @@ psString query = pxDataGet("pstamptool_project.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -883,7 +938,7 @@ char *query = psStringCopy ("UPDATE pstampProject SET"); - + psStringAppend(&query, " state = '%s'", state); - + psStringAppend(&query, " WHERE proj_id = %" PRId64, proj_id); @@ -896,5 +951,5 @@ psU64 affected = psDBAffectedRows(config->dbh); if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); return false; @@ -910,14 +965,37 @@ PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); PXOPT_LOOKUP_STR(imagedb, config->args, "-imagedb", true, false); - PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", false, false); - PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", true, false); + PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false); PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); - psString query = pxDataGet("pstamptool_pendingdependent.sql"); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); + + // start a transaction eraly so it will contain any row level locks + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString query = pxDataGet("pstamptool_getdependent.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (!no_create) { + // This will lock the row until the transaction is committed + psStringAppend(&query, " AND %s FOR UPDATE", whereClause); + } else { + psStringAppend(&query, " AND %s", whereClause); + } + psFree(whereClause); + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -937,13 +1015,24 @@ if (!dep_id) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); return false; } printf("%" PRId64 "\n", dep_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return true; } if (no_create) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; } // no existing dependent that matches, insert one + // Since we have multiple processes running jobs we have a + // race condition here so that's why we need to lock the table if (!pstampDependentInsert( @@ -953,13 +1042,35 @@ stage, stage_id, + component, imagedb, rlabel, - no_magic + need_magic, + 0 // fault )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } psError(PS_ERR_UNKNOWN, false, "failed to insert pstampDependent"); return false; } + // if we try and get this after commit zero is returned psS64 dep_id = psDBLastInsertID(config->dbh); + if (!dep_id) { + psError(PS_ERR_UNKNOWN, false, "psDBLastInsertID returned NULL"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } printf("%" PRId64 "\n", dep_id); @@ -973,4 +1084,9 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-rlabel", "rlabel", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -979,10 +1095,10 @@ psString query = pxDataGet("pstamptool_pendingdependent.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampDependent"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -1031,11 +1147,24 @@ PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); - - char *query ="UPDATE pstampDependent" - " SET state = '%s'" - " WHERE dep_id = %" PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, dep_id)) { + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + if (!fault && !state) { + psError(PS_ERR_UNKNOWN, true, "at least one of -set_state or fault is required"); + return false; + } + psString query = psStringCopy("UPDATE pstampDependent SET"); + bool needComma = false; + if (state) { + psStringAppend(&query, " state = '%s'", state); + needComma = true; + } + if (fault) { + psStringAppend(&query, "%s fault = %d", needComma ? ", " : "", fault); + needComma = true; + } + psStringAppend(&query, " WHERE dep_id = %" PRId64, dep_id); + + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1045,5 +1174,5 @@ psU64 affected = psDBAffectedRows(config->dbh); if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); return false; @@ -1052,2 +1181,35 @@ return true; } +static bool revertdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fault", "pstampDependent.fault", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = pxDataGet("pstamptool_revertdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} Index: branches/tap_branches/ippTools/src/pstamptool.h =================================================================== --- branches/tap_branches/ippTools/src/pstamptool.h (revision 25900) +++ branches/tap_branches/ippTools/src/pstamptool.h (revision 27838) @@ -46,4 +46,5 @@ PSTAMPTOOL_MODE_PENDINGDEPENDENT, PSTAMPTOOL_MODE_UPDATEDEPENDENT, + PSTAMPTOOL_MODE_REVERTDEPENDENT, } pstamptoolMode; Index: branches/tap_branches/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/pstamptoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/pstamptoolConfig.c (revision 27838) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psError(psErrorCodeLast(), false, "Can't find site configuration!\n"); psFree(config); return NULL; @@ -45,30 +45,38 @@ // -adddatastore psMetadata *adddatastoreArgs = psMetadataAlloc(); - psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-uri", 0, "define storage uri (required)", NULL); - psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-out_product", 0, "define output product name (required)", NULL); - psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-last_fileset", 0, "define last fileset seen", NULL); - psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-state", 0, "define datastore state", "enabled"); - + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_uri", 0, "define storage uri (required)", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_out_product", 0, "define output product name (required)", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_last_fileset", 0, "define last fileset seen", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_state", 0, "define datastore state", "enabled"); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_label", 0, "define datastore label", NULL); + psMetadataAddS32(adddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 60); + // -datastore psMetadata *datastoreArgs = psMetadataAlloc(); - psMetadataAddS64(datastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddS64(datastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddBool(datastoreArgs, PS_LIST_TAIL, "-ready", 0, "list data stores ready to be polled", false); psMetadataAddBool(datastoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -moddatastore psMetadata *moddatastoreArgs = psMetadataAlloc(); - psMetadataAddS64(moddatastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); - psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-last_fileset", 0, "set last_fileset seen", NULL); - psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddS64(moddatastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_last_fileset", 0, "set last_fileset seen", NULL); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddS32(moddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 0); + psMetadataAddBool(moddatastoreArgs, PS_LIST_TAIL, "-update_timestamp", 0, "update the timestamp", false); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); // -addreq psMetadata *addreqArgs = psMetadataAlloc(); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); - psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0, "define request name", NULL); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0, "define request label", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0, "define request name", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0, "define request label", NULL); // -pendingreq psMetadata *pendingreqArgs = psMetadataAlloc(); - psMetadataAddS64(pendingreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddS64(pendingreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddStr(pendingreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); psMetadataAddU64(pendingreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -76,7 +84,7 @@ // -listreq psMetadata *listreqArgs = psMetadataAlloc(); - psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "list by req_id", 0); - psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0, "list by name", NULL); - psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0, "req_id to not list", 0); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "list by req_id", 0); + psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0, "list by name", NULL); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0, "req_id to not list", 0); psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -84,4 +92,5 @@ // -completedreq psMetadata *completedreqArgs = psMetadataAlloc(); + psMetadataAddStr(completedreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddU64(completedreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(completedreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -89,35 +98,36 @@ // -updatereq psMetadata *updatereqArgs = psMetadataAlloc(); - psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); + psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); // -revertreq psMetadata *revertreqArgs = psMetadataAlloc(); - psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); - psMetadataAddS16(revertreqArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); - psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); + psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS16(revertreqArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); + psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); // -addjob psMetadata *addjobArgs = psMetadataAlloc(); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "define state", "run"); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure id", 0); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); - psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "define state", "run"); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure id", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); + psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); // -listjob psMetadata *listjobArgs = psMetadataAlloc(); - psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); - psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); - psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); + psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -125,6 +135,7 @@ // -pendingjob psMetadata *pendingjobArgs = psMetadataAlloc(); - psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); - psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); + psMetadataAddStr(pendingjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddU64(pendingjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -132,14 +143,16 @@ // -updatejob psMetadata *updatejobArgs = psMetadataAlloc(); - psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); + psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id for which to change state", 0); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); + psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", 0); // -revertjob psMetadata *revertjobArgs = psMetadataAlloc(); - psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); - psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id_min", 0, "minimum req_id to revert", 0); - psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id to revert", 0); - psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id_min", 0, "minimum req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id to revert", 0); + psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddStr(revertjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted jobs", false); psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -147,24 +160,32 @@ // -getdependent psMetadata *getdependentArgs = psMetadataAlloc(); - psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run (required)", 0); - psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run (required)", NULL); - psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run (required)", NULL); - psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); - psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); - psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent (required)", 0); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for depenent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent ", NULL); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-need_magic", 0, "define need_magic", false); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no matching dependent do not create one", false); // -updatedependent psMetadata *updatedependentArgs = psMetadataAlloc(); - psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent run (required)", 0); - psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); + psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent (required)", 0); + psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); + psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-fault", 0, "new value for fault", 0); + + // -revertdependent + psMetadata *revertdependentArgs = psMetadataAlloc(); + psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent", 0); + psMetadataAddStr(revertdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for dependent ", NULL); + psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault", 0, "define fault", 0); // -pendingdependent psMetadata *pendingdependentArgs = psMetadataAlloc(); - psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run", 0); - psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run", NULL); - psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run", NULL); - psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); - psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); - psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent", 0); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for dependent", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for pstampRequest", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent", NULL); psMetadataAddU64(pendingdependentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingdependentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -182,5 +203,5 @@ // -modproject psMetadata *modprojectArgs = psMetadataAlloc(); - psMetadataAddS64(modprojectArgs, PS_LIST_TAIL, "-proj_id", 0, "define project ID to modify (required)", 0); + psMetadataAddS64(modprojectArgs, PS_LIST_TAIL, "-proj_id", 0, "define project ID to modify (required)", 0); psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-imagedb", 0, "define name of database for project", NULL); psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-state", 0, "define state for project (enabled, disabled)", NULL); @@ -192,5 +213,5 @@ // -project psMetadata *projectArgs = psMetadataAlloc(); - psMetadataAddStr(projectArgs, PS_LIST_TAIL, "-name", 0, "define project name to list (required)", NULL); + psMetadataAddStr(projectArgs, PS_LIST_TAIL, "-name", 0, "define project name to list (required)", NULL); psMetadataAddBool(projectArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -218,4 +239,5 @@ PXOPT_ADD_MODE("-updatedependent", "", PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentArgs); PXOPT_ADD_MODE("-pendingdependent","", PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentArgs); + PXOPT_ADD_MODE("-revertdependent","", PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentArgs); PXOPT_ADD_MODE("-addproject", "", PSTAMPTOOL_MODE_ADDPROJECT, addprojectArgs); Index: branches/tap_branches/ippTools/src/pubtool.c =================================================================== --- branches/tap_branches/ippTools/src/pubtool.c (revision 25900) +++ branches/tap_branches/ippTools/src/pubtool.c (revision 27838) @@ -32,4 +32,5 @@ static bool defineclientMode(pxConfig *config); +static bool updateclientMode(pxConfig *config); static bool definerunMode(pxConfig *config); static bool pendingMode(pxConfig *config); @@ -57,4 +58,5 @@ switch (config->mode) { MODECASE(PUBTOOL_MODE_DEFINECLIENT, defineclientMode); + MODECASE(PUBTOOL_MODE_UPDATECLIENT, updateclientMode); MODECASE(PUBTOOL_MODE_DEFINERUN, definerunMode); MODECASE(PUBTOOL_MODE_PENDING, pendingMode); @@ -93,9 +95,52 @@ // optional PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); - - if (!publishClientInsert(config->dbh, 0, product, stage, workdir, comment)) { - psError(PS_ERR_UNKNOWN, false, "Database error"); - return false; - } + PXOPT_LOOKUP_BOOL(unmagicked, config->args, "-unmagicked", false); + + if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + return true; +} + +static bool updateclientMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-product", "product", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); + + PXOPT_LOOKUP_BOOL(active, config->args, "-active", false); + PXOPT_LOOKUP_BOOL(inactive, config->args, "-inactive", false); + + if ((!active && !inactive) || (active && inactive)) { + psError(PS_ERR_UNKNOWN, false, "Must specify one, and only one, of -active and -inactive."); + psFree(where); + return false; + } + + psString query = NULL; // Query to run + psStringAppend(&query, "UPDATE publishClient SET active = %d", active ? 1 : 0); + + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\n WHERE %s", clause); + psFree(clause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + return false; + } + psFree(query); + + long numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("pubtool", PS_LOG_INFO, "%ld rows updated.", numUpdated); return true; @@ -112,11 +157,9 @@ // optional PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); // define using newExp label psString query = pxDataGet("pubtool_definerun.sql"); // Query to run if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -167,4 +210,5 @@ psS64 client = psMetadataLookupS64(NULL, row, "client_id"); // Client identifier psS64 stage = psMetadataLookupS64(NULL, row, "stage_id"); // Stage identifier + char *label = psMetadataLookupStr(NULL, row, "src_label"); // label from correct source if (!publishRunInsert(config->dbh, 0, client, stage, label, "new")) { @@ -205,5 +249,5 @@ psString query = pxDataGet("pubtool_pending.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -311,9 +355,9 @@ PXOPT_COPY_STR(config->args, where, "-client_id", "publishClient.client_id", "=="); PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-label", "publishRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-label", "publishRun.label", "=="); psString query = pxDataGet("pubtool_revert.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; Index: branches/tap_branches/ippTools/src/pubtool.h =================================================================== --- branches/tap_branches/ippTools/src/pubtool.h (revision 25900) +++ branches/tap_branches/ippTools/src/pubtool.h (revision 27838) @@ -26,4 +26,5 @@ PUBTOOL_MODE_NONE = 0x0, PUBTOOL_MODE_DEFINECLIENT, + PUBTOOL_MODE_UPDATECLIENT, PUBTOOL_MODE_DEFINERUN, PUBTOOL_MODE_PENDING, Index: branches/tap_branches/ippTools/src/pubtoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/pubtoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/pubtoolConfig.c (revision 27838) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!"); + psError(psErrorCodeLast(), false, "Can't find site configuration!"); psFree(config); return NULL; @@ -49,9 +49,19 @@ psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + psMetadataAddBool(defineclientArgs, PS_LIST_TAIL, "-unmagicked", 0, "allow unmagicked data?", false); + + // -updateclient + psMetadata *updateclientArgs = psMetadataAlloc(); + psMetadataAddS64(updateclientArgs, PS_LIST_TAIL, "-client_id", 0, "search by client_id", 0); + psMetadataAddStr(updateclientArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); + psMetadataAddStr(updateclientArgs, PS_LIST_TAIL, "-product", 0, "search by product", NULL); + psMetadataAddStr(updateclientArgs, PS_LIST_TAIL, "-comment", 0, "search by comment (LIKE)", NULL); + psMetadataAddBool(updateclientArgs, PS_LIST_TAIL, "-active", 0, "set to active state", NULL); + psMetadataAddBool(updateclientArgs, PS_LIST_TAIL, "-inactive", 0, "set to inactive state", NULL); // -definerun psMetadata *definerunArgs = psMetadataAlloc(); psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-client_id", 0, "search by client_id", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "set and search by label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "set and search by label", NULL); // -pending @@ -78,5 +88,5 @@ psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", NULL); psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); - psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on label", NULL); @@ -85,4 +95,5 @@ PXOPT_ADD_MODE("-defineclient", "", PUBTOOL_MODE_DEFINECLIENT, defineclientArgs); + PXOPT_ADD_MODE("-updateclient", "", PUBTOOL_MODE_UPDATECLIENT, updateclientArgs); PXOPT_ADD_MODE("-definerun", "", PUBTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-pending", "", PUBTOOL_MODE_PENDING, pendingArgs); Index: branches/tap_branches/ippTools/src/pxadd.c =================================================================== --- branches/tap_branches/ippTools/src/pxadd.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxadd.c (revision 27838) @@ -124,11 +124,11 @@ bool pxaddQueueByCamID(pxConfig *config, psS64 cam_id, - char *workdir, - char *reduction, - char *label, + char *workdir, + char *reduction, + char *label, char *data_group, - char *dvodb, + char *dvodb, char *note, - bool image_only) + bool image_only) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -140,5 +140,5 @@ psMemSetPersistent(query, true); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -149,14 +149,14 @@ // on 32/64 if (!p_psDBRunQueryF(config->dbh, query, - "new", // state - workdir ? workdir : "NULL", - "dirty", //workdir_state - reduction? reduction : "NULL", - label ? label : "NULL", - data_group ? data_group : "NULL", - dvodb ? dvodb : "NULL", - note ? note : "NULL", - image_only, - (long long) cam_id + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + reduction? reduction : "NULL", + label ? label : "NULL", + data_group ? data_group : "NULL", + dvodb ? dvodb : "NULL", + note ? note : "NULL", + image_only, + (long long) cam_id )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/tap_branches/ippTools/src/pxadmin.c =================================================================== --- branches/tap_branches/ippTools/src/pxadmin.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxadmin.c (revision 27838) @@ -94,5 +94,5 @@ psString query = pxDataGet("pxadmin_create_tables.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -114,5 +114,5 @@ psFree(query); - if (!insert_dbversion(config, PACKAGE_VERSION)) { + if (!insert_dbversion(config, IPPDB_VERSION)) { psError(PS_ERR_UNKNOWN, false, "failed to set database version"); return false; @@ -135,5 +135,5 @@ psString query = pxDataGet("pxadmin_create_mirror_tables.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -155,5 +155,5 @@ psFree(query); - if (!insert_dbversion(config, PACKAGE_VERSION)) { + if (!insert_dbversion(config, IPPDB_VERSION)) { psError(PS_ERR_UNKNOWN, false, "failed to set database version"); return false; @@ -220,5 +220,5 @@ psString query = pxDataGet("pxadmin_drop_tables.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/pxcam.c =================================================================== --- branches/tap_branches/ippTools/src/pxcam.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxcam.c (revision 27838) @@ -76,16 +76,16 @@ bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where) { - PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); - PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); @@ -152,4 +152,5 @@ char *tess_id, char *end_stage, + psS64 magicked, char *note) { @@ -162,5 +163,5 @@ psMemSetPersistent(query, true); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -168,5 +169,5 @@ // queue the exp - // XXX chip_id is being cast here work around psS64 have a different type + // XXX chip_id and magicked are being cast here work around psS64 have a different type // different on 32/64 if (!p_psDBRunQueryF(config->dbh, query, @@ -182,4 +183,5 @@ tess_id ? tess_id : "NULL", end_stage ? end_stage : "NULL", + (long long) magicked, note ? note : "NULL", (long long)chip_id Index: branches/tap_branches/ippTools/src/pxcam.h =================================================================== --- branches/tap_branches/ippTools/src/pxcam.h (revision 25900) +++ branches/tap_branches/ippTools/src/pxcam.h (revision 27838) @@ -40,4 +40,5 @@ char *tess_id, char *end_stage, + psS64 magicked, char *note); Index: branches/tap_branches/ippTools/src/pxchip.c =================================================================== --- branches/tap_branches/ippTools/src/pxchip.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxchip.c (revision 27838) @@ -96,5 +96,5 @@ PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); @@ -125,9 +125,9 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); - return true; -} - -bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked) + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + return true; +} + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -140,5 +140,5 @@ } - char *query = "UPDATE chipRun SET state = '%s', magicked = %d WHERE chip_id = %" PRId64; + char *query = "UPDATE chipRun SET state = '%s', magicked = %" PRId64 " WHERE chip_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, state, magicked, chip_id)) { psError(PS_ERR_UNKNOWN, false, @@ -174,4 +174,9 @@ psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'"); + + if (!strcmp(state, "cleaned") || !strcmp(state, "purged")) { + // if magicked is non-zero set it to -1 + psStringAppend(&query, ", chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"); + } if (where && psListLength(where->list) > 0) { @@ -213,5 +218,5 @@ /* } */ - psString query = psStringCopy("UPDATE chipProcessedImfile JOIN chipRun USING(chip_id) JOIN rawExp ON chipRun.exp_id = rawExp.exp_id SET state = '%s'"); + psString query = psStringCopy("UPDATE chipProcessedImfile JOIN chipRun USING(chip_id) JOIN rawExp ON chipRun.exp_id = rawExp.exp_id SET data_state = '%s'"); if (where && psListLength(where->list) > 0) { @@ -219,4 +224,7 @@ psStringAppend(&query, " %s", whereClause); psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; } Index: branches/tap_branches/ippTools/src/pxchip.h =================================================================== --- branches/tap_branches/ippTools/src/pxchip.h (revision 25900) +++ branches/tap_branches/ippTools/src/pxchip.h (revision 27838) @@ -27,5 +27,5 @@ -bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked); +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked); bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); Index: branches/tap_branches/ippTools/src/pxdqstats.c =================================================================== --- branches/tap_branches/ippTools/src/pxdqstats.c (revision 27838) +++ branches/tap_branches/ippTools/src/pxdqstats.c (revision 27838) @@ -0,0 +1,103 @@ +/* pxdqstats.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdqstats.h" + +bool pxdqstatsSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + + return true; +} + +bool pxdqstatsGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + + return true; +} Index: branches/tap_branches/ippTools/src/pxdqstats.h =================================================================== --- branches/tap_branches/ippTools/src/pxdqstats.h (revision 27838) +++ branches/tap_branches/ippTools/src/pxdqstats.h (revision 27838) @@ -0,0 +1,15 @@ +/* pxdqstats.h + */ + +#ifndef PXDQSTATS_H +#define PXDQSTATS_H 1 + +#include + +#include "pxtools.h" + +bool pxdqstatsSetSearchArgs (psMetadata *md); +bool pxdqstatsGetSearchArgs (pxConfig *config, psMetadata *where); + + +#endif // PXDQSTATS_H Index: branches/tap_branches/ippTools/src/pxerrors.c =================================================================== --- branches/tap_branches/ippTools/src/pxerrors.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxerrors.c (revision 27838) @@ -29,13 +29,11 @@ switch (err) { case PXTOOLS_ERR_SYS: - return PS_EXIT_SYS_ERROR; + return PS_EXIT_SYS_ERROR; case PXTOOLS_ERR_CONFIG: - return PS_EXIT_CONFIG_ERROR; + return PS_EXIT_CONFIG_ERROR; case PXTOOLS_ERR_PROG: - return PS_EXIT_PROG_ERROR; - case PXTOOLS_ERR_DATA: - return PS_EXIT_DATA_ERROR; + return PS_EXIT_PROG_ERROR; default: - return PS_EXIT_UNKNOWN_ERROR; + return PS_EXIT_UNKNOWN_ERROR; } return PS_EXIT_UNKNOWN_ERROR; Index: branches/tap_branches/ippTools/src/pxfake.c =================================================================== --- branches/tap_branches/ippTools/src/pxfake.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxfake.c (revision 27838) @@ -74,5 +74,5 @@ psMemSetPersistent(query, true); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/pxmagic.c =================================================================== --- branches/tap_branches/ippTools/src/pxmagic.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxmagic.c (revision 27838) @@ -35,5 +35,5 @@ psString query_temp = pxDataGet(queryFile); if (!query_temp) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement from %s", queryFile); psFree(queryFile); if (!psDBRollback(config->dbh)) { @@ -75,9 +75,9 @@ if (not_destreaked) { if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + psError(PXTOOLS_ERR_CONFIG, true, "providing -not_destreaked and -destreaked makes no sense"); return false; } if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + psError(PXTOOLS_ERR_CONFIG, true, "providing -not_destreaked and -magicked makes no sense"); return false; } @@ -87,4 +87,5 @@ psStringAppend(out, " AND %s.magicked != 0", table); } + // Note -magicked is handled by the caller. XXX: Why? return true; } Index: branches/tap_branches/ippTools/src/pxregister.c =================================================================== --- branches/tap_branches/ippTools/src/pxregister.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxregister.c (revision 27838) @@ -58,2 +58,67 @@ return true; } + +bool pxrawExpSetState(pxConfig *config, psS64 exp_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "full", 5) == 0) + || (strncmp(state, "new", 4) == 0) + || (strncmp(state, "goto_compressed", 14) == 0) + || (strncmp(state, "compressed", 11) == 0) + || (strncmp(state, "goto_lossy", 11) == 0) + || (strncmp(state, "lossy", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid rawExp state: %s", state); + return false; + } + + char *query = "UPDATE rawExp SET state = '%s' WHERE exp_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} +bool pxrawImfileSetState(pxConfig *config, psS64 exp_id, const char *class_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "full", 5) == 0) + || (strncmp(state, "new", 4) == 0) + || (strncmp(state, "goto_compressed", 14) == 0) + || (strncmp(state, "compressed", 11) == 0) + || (strncmp(state, "goto_lossy", 11) == 0) + || (strncmp(state, "lossy", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid rawImfile data_state: %s", state); + return false; + } + + char *query = "UPDATE rawImfile SET data_state = '%s' WHERE class_id = %s AND exp_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, class_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} Index: branches/tap_branches/ippTools/src/pxregister.h =================================================================== --- branches/tap_branches/ippTools/src/pxregister.h (revision 25900) +++ branches/tap_branches/ippTools/src/pxregister.h (revision 27838) @@ -26,4 +26,6 @@ bool pxnewExpSetState(pxConfig *config, psS64 exp_id, const char *state); +bool pxrawExpSetState(pxConfig *config, psS64 exp_id, const char *state); +bool pxrawImfileSetState(pxConfig *config, psS64 exp_id, const char *class_id, const char *state); #endif // PXREGISTER_H Index: branches/tap_branches/ippTools/src/pxtools.c =================================================================== --- branches/tap_branches/ippTools/src/pxtools.c (revision 25900) +++ branches/tap_branches/ippTools/src/pxtools.c (revision 27838) @@ -61,5 +61,5 @@ if (!state) { psError(PS_ERR_PROGRAMMING, false, "%s not found in row %ld of table %s", - columnName, i, tableName); + columnName, i, tableName); return false; } @@ -70,5 +70,5 @@ // if state isn't cleaned or full we can't set it to cleaned psError(PS_ERR_PROGRAMMING, true, "%s with state %s may not be exported cleaned", - tableName, state); + tableName, state); return false; } @@ -90,19 +90,19 @@ psAssert (entry, "%s should at least have a place-holder", name); if (entry->data.str) { - psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true); - psMetadataItem *item = NULL; - while ((item = psListGetAndIncrement(iter))) { - // need to change the name and comment - psFree (item->name); - item->name = psStringCopy (field); - psFree (item->comment); - item->comment = psStringCopy (op); - if (!psMetadataAddItem(where, item, PS_LIST_TAIL, PS_META_DUPLICATE_OK)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item %s", field); - psFree(where); - return false; - } - } - psFree (iter); + psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true); + psMetadataItem *item = NULL; + while ((item = psListGetAndIncrement(iter))) { + // need to change the name and comment + psFree (item->name); + item->name = psStringCopy (field); + psFree (item->comment); + item->comment = psStringCopy (op); + if (!psMetadataAddItem(where, item, PS_LIST_TAIL, PS_META_DUPLICATE_OK)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", field); + psFree(where); + return false; + } + } + psFree (iter); } return true; @@ -110,5 +110,5 @@ // shared code for updating the various strings for a Run -bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString table, bool has_dist_group) +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -116,9 +116,12 @@ PS_ASSERT_PTR_NON_NULL(pQuery, false); PS_ASSERT_PTR_NON_NULL(*pQuery, false); + PS_ASSERT_PTR_NON_NULL(runTable, false); + PS_ASSERT_PTR_NON_NULL(idColumn, false); + PS_ASSERT_PTR_NON_NULL(fileTable, false); // make sure that -state is not the only selection parameter PXOPT_LOOKUP_STR(where_state, config->args, "-state", false, false); if (where_state && (psListLength(where->list) < 2)) { - psError(PXTOOLS_ERR_DATA, true, "selection by -state alone is not allowed"); + psError(PXTOOLS_ERR_CONFIG, true, "selection by -state alone is not allowed"); return false; } @@ -136,10 +139,10 @@ if ((!state) && (!label) && (!data_group) && (has_dist_group && !dist_group) && !(note)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "parameters are required"); return false; } if (state && ! pxIsValidState(state)) { - psError(PXTOOLS_ERR_DATA, false, "pxIsValidState failed"); + psError(PXTOOLS_ERR_CONFIG, false, "pxIsValidState failed"); return false; } @@ -158,17 +161,34 @@ } while (0) - addColumn(table, state); - addColumn(table, data_group); + addColumn(runTable, state); + addColumn(runTable, data_group); if (has_dist_group) { - addColumn(table, dist_group); - } - addColumn(table, note); - addColumn(table, label); + addColumn(runTable, dist_group); + } + addColumn(runTable, note); + addColumn(runTable, label); + + psString joinHook = psStringCopy(""); + psString fileWhere = NULL; + if (state && !strcmp(state, "update")) { + psStringAppend(&joinHook, "\n JOIN %s USING(%s)", fileTable, idColumn); + psStringAppend(pQuery, ", %s.data_state = 'update'", fileTable); + psStringAppend(&fileWhere, "AND %s.data_state = 'cleaned'", fileTable); + } psString whereClause = psDBGenerateWhereSQL(where, NULL); psStringAppend(pQuery, " %s", whereClause); psFree(whereClause); - - if (!p_psDBRunQuery(config->dbh, *pQuery)) { + if (fileWhere) { + psStringAppend(pQuery, "%s", fileWhere); + } + + bool mdok; // Status of MD lookup + if (psMetadataLookupBool(&mdok, config->args, "-pretend")) { + psLogMsg("pxtools", PS_LOG_INFO, "Query to run: %s\n", *pQuery); + return true; + } + + if (!p_psDBRunQueryF(config->dbh, *pQuery, joinHook)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -198,5 +218,5 @@ } if (psArrayLength(output) > 1) { - psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found in dbversion: %" PRId64, + psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found in dbversion: %ld", psArrayLength(output)); return false; @@ -221,5 +241,5 @@ return NULL; } - + psString version = psMetadataLookupStr(NULL, md, "schema_version"); @@ -257,5 +277,5 @@ return false; } - + psMetadataItem *dbversion = psListGet(multi_item->data.list, 0); if (!dbversion) { @@ -272,5 +292,5 @@ return false; } - + psString import_version = psMetadataLookupStr(NULL, md, "schema_version"); if (import_version && strcmp(import_version, schema_version)) { Index: branches/tap_branches/ippTools/src/pxtools.h =================================================================== --- branches/tap_branches/ippTools/src/pxtools.h (revision 25900) +++ branches/tap_branches/ippTools/src/pxtools.h (revision 27838) @@ -65,5 +65,5 @@ bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets); -bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString table, bool has_dist_group); +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group); #define PXOPT_ADD_MODE(option, comment, modeval, argset) \ Index: branches/tap_branches/ippTools/src/pxtoolsErrorCodes.dat =================================================================== --- branches/tap_branches/ippTools/src/pxtoolsErrorCodes.dat (revision 25900) +++ branches/tap_branches/ippTools/src/pxtoolsErrorCodes.dat (revision 27838) @@ -8,3 +8,2 @@ CONFIG Problem in configure files PROG Programming error -DATA invalid data Index: branches/tap_branches/ippTools/src/pztool.c =================================================================== --- branches/tap_branches/ippTools/src/pztool.c (revision 25900) +++ branches/tap_branches/ippTools/src/pztool.c (revision 27838) @@ -117,5 +117,6 @@ telescope, uri, - NULL // epoch + NULL, // epoch + 0 // use_compress )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -304,5 +305,5 @@ psArray *cameras = pzGetPendingCameras(config); if (!cameras) { - psError(PXTOOLS_ERR_DATA, false, "failed to find any cameras"); + psError(PXTOOLS_ERR_CONFIG, false, "failed to find any cameras"); return false; } @@ -314,5 +315,5 @@ psString query = pxDataGet("pztool_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(cameraImfiles); return false; @@ -492,5 +493,5 @@ psString query = pxDataGet("pztool_find_completed_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -783,5 +784,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); @@ -807,5 +808,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); @@ -814,5 +815,5 @@ psString query = pxDataGet("pztool_revertcopied.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -848,5 +849,5 @@ psString query = pxDataGet("pztool_revert_downloadimfile_faults.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -863,5 +864,5 @@ psString query = pxDataGet("pztool_revert_fileset_faults.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/pztoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/pztoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/pztoolConfig.c (revision 27838) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psError(psErrorCodeLast(), false, "Can't find site configuration!\n"); psFree(config); return NULL; Index: branches/tap_branches/ippTools/src/receivetool.c =================================================================== --- branches/tap_branches/ippTools/src/receivetool.c (revision 25900) +++ branches/tap_branches/ippTools/src/receivetool.c (revision 27838) @@ -105,4 +105,5 @@ // optional PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(last, config->args, "-last", false, false); PXOPT_LOOKUP_STR(status_product, config->args, "-status_product", false, false); @@ -110,5 +111,5 @@ PXOPT_LOOKUP_STR(ds_dbhost, config->args, "-ds_dbhost", false, false); - if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, comment, last, status_product, ds_dbname, ds_dbhost)) { + if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, state, comment, last, status_product, ds_dbname, ds_dbhost)) { psError(PS_ERR_UNKNOWN, false, "Database error"); return false; @@ -135,5 +136,5 @@ psString query = pxDataGet("receivetool_list.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -142,5 +143,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -194,5 +195,5 @@ psString query = pxDataGet("receivetool_addfileset.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); return false; } @@ -301,5 +302,5 @@ psString query = pxDataGet("receivetool_pendingfileset.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -380,5 +381,5 @@ psString file_type = psMetadataLookupStr(NULL, md, "file_type"); psString component = psMetadataLookupStr(NULL, md, "component"); - + if (!file) { psError(PS_ERR_UNKNOWN, false, "failed to find value for file"); @@ -430,5 +431,5 @@ psString query = pxDataGet("receivetool_pendingfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -513,5 +514,5 @@ psString query = pxDataGet("receivetool_revert.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -546,5 +547,5 @@ psString query = pxDataGet("receivetool_toadvance.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -612,5 +613,5 @@ psString query = NULL; // Query to execute psStringAppend(&query, "UPDATE receiveFileset SET "); - + psString sep = ""; if (fault) { @@ -630,5 +631,5 @@ sep = ","; } - + psStringAppend(&query, " WHERE fileset_id = %" PRId64, fileset_id); Index: branches/tap_branches/ippTools/src/receivetoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/receivetoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/receivetoolConfig.c (revision 27838) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!"); + psError(psErrorCodeLast(), false, "Can't find site configuration!"); psFree(config); return NULL; @@ -49,4 +49,5 @@ psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-state", 0, "define state", "enabled"); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-last", 0, "define last fileset", NULL); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-status_product", 0, "define status_product", NULL); Index: branches/tap_branches/ippTools/src/regtool.c =================================================================== --- branches/tap_branches/ippTools/src/regtool.c (revision 25900) +++ branches/tap_branches/ippTools/src/regtool.c (revision 27838) @@ -37,4 +37,5 @@ static bool revertprocessedimfileMode(pxConfig *config); static bool updateprocessedimfileMode(pxConfig *config); +static bool pendingcompressimfileMode(pxConfig *config); // exp static bool pendingexpMode(pxConfig *config); @@ -44,4 +45,8 @@ static bool updateprocessedexpMode(pxConfig *config); static bool cleardupexpMode(pxConfig *config); +static bool finishcompressexpMode(pxConfig *config); + +static bool updatebyqueryMode(pxConfig *config); + static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -77,4 +82,7 @@ MODECASE(REGTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(REGTOOL_MODE_UPDATEBYQUERY, updatebyqueryMode); + MODECASE(REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileMode); + MODECASE(REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpMode); MODECASE(REGTOOL_MODE_CLEARDUPEXP, cleardupexpMode); MODECASE(REGTOOL_MODE_EXPORTRUN, exportrunMode); @@ -117,5 +125,5 @@ psString query = pxDataGet("regtool_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -189,4 +197,5 @@ PXOPT_LOOKUP_STR(obs_mode, config->args, "-obs_mode", false, false); PXOPT_LOOKUP_STR(obs_group, config->args, "-obs_group", false, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false); PXOPT_LOOKUP_F32(airmass, config->args, "-airmass", false, false); @@ -255,4 +264,5 @@ class_id, uri, + data_state, exp_type, filelevel, @@ -333,11 +343,15 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "=="); - PXOPT_COPY_S64(config->args, where, "-magicked", "magicked", "=="); + PXOPT_COPY_STR(config->args, where, "-data_state", "data_state", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(allfiles, config->args, "-allfiles", false); + if (allfiles) { + faulted = false; + } PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -347,9 +361,9 @@ psString where2 = NULL; if (!pxmagicAddWhere(config, &where2, "rawImfile")) { - psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + psError(psErrorCodeLast(), false, "pxmagicAddWhere failed"); return false; } if (!pxspaceAddWhere(config, &where2, "rawImfile")) { - psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + psError(psErrorCodeLast(), false, "pxspaceAddWhere failed"); return false; } @@ -357,5 +371,5 @@ psString query = pxDataGet("regtool_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -367,5 +381,5 @@ psFree(whereClause); } else if (!all && !where2) { - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -379,5 +393,5 @@ // list only faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault != 0"); - } else { + } else if (!allfiles) { // don't list faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); @@ -452,5 +466,5 @@ psString query = pxDataGet("regtool_revertprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -463,5 +477,5 @@ } else { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -491,12 +505,14 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); -/* PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); */ PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); - - if ((fault == INT16_MAX) && !isfinite(burntool_state)) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state must be selected"); - return false; - } - if ((fault != INT16_MAX) && isfinite(burntool_state)) { + PXOPT_LOOKUP_S32(set_bytes, config->args, "-set_bytes", false, false); + PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + + if ((fault == INT16_MAX) && !isfinite(burntool_state) && !(set_state)) { + psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); + return false; + } + if ((fault != INT16_MAX) && isfinite(burntool_state)) { psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); return false; @@ -508,29 +524,141 @@ if (fault != INT16_MAX) { - // this is fairly dangerous : can set all if the where is not set... - if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - psFree (where); - return false; - } - } - - if (isfinite(burntool_state)) { - psString query = pxDataGet("regtool_updateprocessedimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, burntool_state, exp_id, class_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); + // this is fairly dangerous : can set all if the where is not set... + if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return(true); } psFree (where); + psString setvalues = NULL; + if (burntool_state != INT16_MAX) { + psStringAppend(&setvalues,"rawImfile.burntool_state = %d",burntool_state); + } + if (set_bytes != INT32_MAX) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.bytes = %d",set_bytes); + } + if (set_md5sum) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.md5sum = '%s'",set_md5sum); + } + if (set_state) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state); + } + + psString query = pxDataGet("regtool_updateprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // printf(query,setvalues,exp_id,class_id); + if (!p_psDBRunQueryF(config->dbh, query, setvalues, exp_id,class_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(setvalues); + return false; + } + psFree(setvalues); + psFree(query); + return true; +} + +static bool pendingcompressimfileMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (whereClause) { + psStringPrepend(&whereClause, "\n AND "); + } + PXOPT_LOOKUP_BOOL(compress, config->args, "-compress", false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + if ((compress && clean) || (!compress && !clean)) { + psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')\n OR (data_state = 'goto_lossy' AND state = 'goto_lossy')) "); + } + else if (compress) { + psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')) "); + } + else if (clean) { + psStringAppend(&whereClause, "\n AND ((data_state = 'goto_lossy' AND state = 'goto_lossy')) "); + } + else { + psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')\n OR (data_state = 'goto_lossy' AND state = 'goto_lossy')) "); + } + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_pendingcompressimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + // printf(query,whereClause,limitString); + if (!p_psDBRunQueryF(config->dbh, query, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingCompressImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; } @@ -549,5 +677,5 @@ psString query = pxDataGet("regtool_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -626,4 +754,5 @@ PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); @@ -687,5 +816,5 @@ psString query = pxDataGet("regtool_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -787,4 +916,5 @@ filelevel, workdir, + state, reduction, dvodb, @@ -920,5 +1050,5 @@ psString query = pxDataGet("regtool_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -930,11 +1060,12 @@ PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">="); PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<"); @@ -965,4 +1096,14 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxmagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -970,28 +1111,33 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); } psFree(where); + if (where2) { + psStringAppend(&query, " %s ", where2); + psFree(where2); + } + if (faulted) { - // list only faulted rows - psStringAppend(&query, " %s", "AND rawExp.fault != 0"); + // list only faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault != 0"); } else { - // don't list faulted rows - psStringAppend(&query, " %s", "AND rawExp.fault = 0"); + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault = 0"); } // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; } psFree(query); @@ -999,29 +1145,29 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; } if (!psArrayLength(output)) { - psTrace("regtool", PS_LOG_INFO, "no rows found"); + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } + return false; + } } @@ -1033,42 +1179,42 @@ static bool revertprocessedexpMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id_begin", "exp_id", ">="); - PXOPT_COPY_S64(config->args, where, "-exp_id_end", "exp_id", "<="); - - psString query = pxDataGet("regtool_revertprocessedexp.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - psFree(where); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } else { - psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id_begin", "exp_id", ">="); + PXOPT_COPY_S64(config->args, where, "-exp_id_end", "exp_id", "<="); + + psString query = pxDataGet("regtool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - - int numUpdated = psDBAffectedRows(config->dbh); - - psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated); - - return true; + return false; + } + psFree(query); + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated); + + return true; } @@ -1076,20 +1222,207 @@ static bool updateprocessedexpMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - - if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - psFree(where); - return false; - } + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree(where); - - return true; -} + return false; + } + psFree(where); + + return true; +} + + +static bool finishcompressexpMode(pxConfig *config) { + + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_finishcompressexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " %s ", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s ", limitString); + psFree(limitString); + } + // printf(query); + // return(false); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + rawExpRow *row = rawExpObjectFromMetadata(md); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (strcmp(row->state,"goto_compressed") == 0) { + if (!pxrawExpSetState(config,row->exp_id, "compressed")) { + psError(PS_ERR_UNKNOWN, false, "failed to set rawExp.state for exp_id: %" PRId64, row->exp_id); + psFree(row); + psFree(output); + return(false); + } + } + else if (strcmp(row->state,"goto_lossy") == 0) { + if (!pxrawExpSetState(config,row->exp_id, "lossy")) { + psError(PS_ERR_UNKNOWN, false, "failed to set rawExp.state for exp_id: %" PRId64, row->exp_id); + psFree(row); + psFree(output); + return(false); + } + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(row); + } + + psFree(output); + return(true); +} + +static bool updatebyqueryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", true, false); + + // XX test this out; need to make this consistent with the list in regtoolConfig.c + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "rawExp.state", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause,"\n AND "); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + // Update the imfiles first, because if you select by state, you'll clobber things + psString query = pxDataGet("regtool_updatebyqueryimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + + // printf(query,set_state,whereClause,limitString); + if (!p_psDBRunQueryF(config->dbh, query, set_state, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + + psFree(query); + // Now up date the exposure. + query = pxDataGet("regtool_updatebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, set_state,whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + + return true; +} + @@ -1111,5 +1444,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1137,5 +1470,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1218,4 +1551,9 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion to output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); @@ -1229,5 +1567,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1287,4 +1625,10 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + fprintf (stdout, "---- input ----\n"); Index: branches/tap_branches/ippTools/src/regtool.h =================================================================== --- branches/tap_branches/ippTools/src/regtool.h (revision 25900) +++ branches/tap_branches/ippTools/src/regtool.h (revision 27838) @@ -36,4 +36,7 @@ REGTOOL_MODE_UPDATEPROCESSEDEXP, REGTOOL_MODE_CLEARDUPEXP, + REGTOOL_MODE_UPDATEBYQUERY, + REGTOOL_MODE_PENDINGCOMPRESSIMFILE, + REGTOOL_MODE_FINISHCOMPRESSEXP, REGTOOL_MODE_EXPORTRUN, REGTOOL_MODE_IMPORTRUN Index: branches/tap_branches/ippTools/src/regtoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/regtoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/regtoolConfig.c (revision 27838) @@ -43,5 +43,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PXTOOLS_ERR_CONFIG, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -72,6 +72,6 @@ ADD_OPT(Str, addprocessedimfileArgs, "-comment", "define comment ", NULL); ADD_OPT(Str, addprocessedimfileArgs, "-obs_mode", "define observing mode (data usage goal)", NULL); - ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); - + ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-data_state", "define the data_state", "full"); // Note: RA & DEC are supplied here in radians, but the query options use degrees. This // can be justified by the fact that this option (addprocessedimfile) is a software @@ -139,8 +139,10 @@ ADD_OPT(Str, processedimfileArgs, "-obs_mode", "search by obs_mod", NULL); ADD_OPT(Str, processedimfileArgs, "-exp_type", "search by exposure type", NULL); + ADD_OPT(Str, processedimfileArgs, "-data_state", "search by data_state", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL); ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); + ADD_OPT(Bool, processedimfileArgs, "-allfiles", "return imfiles regardless of fault status", false); ADD_OPT(Bool, processedimfileArgs, "-all", "list without search arguments", false); ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); @@ -162,7 +164,11 @@ ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); - ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", 0); + ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", INT16_MAX); ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); - + ADD_OPT(S32, updateprocessedimfileArgs, "-set_bytes", "set bytes", INT32_MAX); + ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum", "set md5sum", NULL); + ADD_OPT(Str, updateprocessedimfileArgs, "-set_state", "set data state", NULL); + + // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); @@ -185,4 +191,5 @@ ADD_OPT(Str, addprocessedexpArgs, "-exp_type", "define exposure type", NULL); ADD_OPT(Str, addprocessedexpArgs, "-workdir", "define the \"default\" workdir for this exposure", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-state", "define the state for this exposure", NULL); ADD_OPT(Str, addprocessedexpArgs, "-dvodb", "define the dvodb for the next processing step", NULL); ADD_OPT(Str, addprocessedexpArgs, "-tess_id", "define the tess_id for the next processing step", NULL); @@ -253,4 +260,5 @@ psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-state", 0, "search by exposure state", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); @@ -285,4 +293,6 @@ psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + pxspaceAddArguments(processedexpArgs); + pxmagicAddArguments(processedexpArgs); psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -301,5 +311,5 @@ psMetadataAddS64(updatedprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", INT16_MAX); - + psMetadataAddStr(updatedprocessedexpArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -312,8 +322,73 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - // -cleardupexp psMetadata *cleardupexpArgs = psMetadataAlloc(); + // -updatebyquery + psMetadata *updatebyqueryArgs = psMetadataAlloc(); + + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "set the state", NULL); + + psMetadataAddS64(updatebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(updatebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(updatebyqueryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-state", 0, "search by rawExp.state", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by obs_mode", NULL); + + psMetadataAddU64(updatebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(updatebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + + // -pendingcompressimfile + psMetadata *pendingcompressimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcompressimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(pendingcompressimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcompressimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-compress", 0, "search only for files to compress", 0); + psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-clean", 0, "search only for files to clean originals", 0); + + // -finishcompressexp + psMetadata *finishcompressexpArgs = psMetadataAlloc(); + psMetadataAddS64(finishcompressexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddBool(finishcompressexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(finishcompressexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -330,4 +405,7 @@ PXOPT_ADD_MODE("-updateprocessedexp", "", REGTOOL_MODE_UPDATEPROCESSEDEXP, updatedprocessedexpArgs); PXOPT_ADD_MODE("-cleardupexp", "", REGTOOL_MODE_CLEARDUPEXP, cleardupexpArgs); + PXOPT_ADD_MODE("-updatebyquery", "", REGTOOL_MODE_UPDATEBYQUERY, updatebyqueryArgs); + PXOPT_ADD_MODE("-pendingcompressimfile", "", REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileArgs); + PXOPT_ADD_MODE("-finishcompressexp", "", REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", REGTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", REGTOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/tap_branches/ippTools/src/stacktool.c =================================================================== --- branches/tap_branches/ippTools/src/stacktool.c (revision 25900) +++ branches/tap_branches/ippTools/src/stacktool.c (revision 27838) @@ -124,4 +124,5 @@ // select based on properties of the raw exposures + PXOPT_COPY_STR(config->args, where, "-select_comment", "rawExp.comment", "LIKE"); PXOPT_COPY_STR(config->args, where, "-select_inst", "rawExp.camera", "=="); PXOPT_COPY_STR(config->args, where, "-select_telescope", "rawExp.telescope", "=="); @@ -145,17 +146,26 @@ PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_min", "camProcessedExp.fwhm_minor", ">="); PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_max", "camProcessedExp.fwhm_minor", "<="); - PXOPT_COPY_F32(config->args, where, "-select_iq_m2_max", "camProcessedExp.iq_m2", "<="); - PXOPT_COPY_F32(config->args, where, "-select_iq_m2_min", "camProcessedExp.iq_m2", ">="); - PXOPT_COPY_F32(config->args, where, "-select_iq_m3_max", "camProcessedExp.iq_m3", "<="); - PXOPT_COPY_F32(config->args, where, "-select_iq_m4_min", "camProcessedExp.iq_m4", ">="); - PXOPT_COPY_F32(config->args, where, "-select_iq_m4_max", "camProcessedExp.iq_m4", "<="); - + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_max", "camProcessedExp.iq_m2", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_min", "camProcessedExp.iq_m2", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m3_max", "camProcessedExp.iq_m3", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_min", "camProcessedExp.iq_m4", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_max", "camProcessedExp.iq_m4", "<="); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "warpSkyfile.skycell_id", "=="); - pxAddLabelSearchArgs (config, where, "-select_label", "warpRun.label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-select_data_group", "warpRun.data_group", "=="); + pxAddLabelSearchArgs (config, where, "-select_label", "warpRun.label", "LIKE"); // define using warp label // these are used to build the HAVING restriction PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">="); + PXOPT_LOOKUP_S32(min_num, config->args, "-min_num", false, false); + if (min_num < 2) { + psError(PXTOOLS_ERR_CONFIG, true, "Require at least two inputs for a stack, but min_num = %d", + min_num); + psFree(where); + psFree(having); + return false; + } // other options applied outside of the WHERE @@ -168,29 +178,29 @@ if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - psString select = pxDataGet("stacktool_definebyquery_part1.sql"); + psFree(having); + return false; + } + + psString select = pxDataGet("stacktool_definebyquery_select.sql"); if (!select) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + psFree(having); + return false; + } + + psString where1 = psStringCopy(""); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&select, " AND %s", whereClause); + psStringAppend(&where1, "\nAND %s", whereClause); psFree(whereClause); } - psString groupby = pxDataGet("stacktool_definebyquery_part2.sql"); - if (!groupby) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - psFree(where); - return false; - } - psStringAppend(&select, " %s", groupby); - psFree(groupby); + psString where2 = psStringCopy(""); + if (label) { + psStringAppend(&where2, "\nWHERE stackRun.label = '%s'", label); + } // Restriction on aggregated quantities using HAVING @@ -224,11 +234,14 @@ psFree(having); - if (!p_psDBRunQuery(config->dbh, select)) { + if (!p_psDBRunQueryF(config->dbh, select, where1, where2)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(select); - psFree(where); + psFree(where1); + psFree(where2); return false; } psFree(select); + psFree(where1); + psFree(where2); psArray *output = p_psDBFetchResult(config->dbh); @@ -272,5 +285,5 @@ } if (!insert) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -287,5 +300,5 @@ psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql"); if (!part2) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(insert); return false; @@ -593,5 +606,5 @@ if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -600,5 +613,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "stackRun", true); + bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true); psFree(query); @@ -663,5 +676,5 @@ psString query = pxDataGet("stacktool_inputskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -669,5 +682,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile"); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); } @@ -737,12 +750,13 @@ psString query = pxDataGet("stacktool_tosum.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psStringCopy(""); // WHERE conditions to add if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + psString new = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\nAND %s", new); + psFree(new); } psFree(where); @@ -755,5 +769,5 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -908,4 +922,5 @@ PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE"); PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); @@ -922,5 +937,5 @@ psString query = pxDataGet("stacktool_sumskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -931,5 +946,5 @@ psFree(whereClause); } else if (!all) { - psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); return false; } @@ -991,10 +1006,10 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -1003,5 +1018,5 @@ psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); if (!delete) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1084,5 +1099,5 @@ psString query = pxDataGet("stacktool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1148,5 +1163,5 @@ psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1209,5 +1224,5 @@ psString query = pxDataGet("stacktool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1313,5 +1328,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/tap_branches/ippTools/src/stacktoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/stacktoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/stacktoolConfig.c (revision 27838) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -59,4 +59,6 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", "object"); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by warpRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_data_group", 0 , "search by warpRun data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_comment", 0, "search for comment (LIKE)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, "search for telescope", NULL); @@ -116,5 +118,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); - + // -addinputskyfile psMetadata *addinputskyfileArgs = psMetadataAlloc(); @@ -181,4 +183,5 @@ #endif psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by stackRun.label", NULL); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-data_group", 0, "search by stackRun.data_group (LIKE comparison)", NULL); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", NULL); psMetadataAddS16(sumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -190,5 +193,5 @@ psMetadata *revertsumskyfileArgs= psMetadataAlloc(); psMetadataAddS64(revertsumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); - psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); psMetadataAddS16(revertsumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddBool(revertsumskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); Index: branches/tap_branches/ippTools/src/staticskytool.c =================================================================== --- branches/tap_branches/ippTools/src/staticskytool.c (revision 27838) +++ branches/tap_branches/ippTools/src/staticskytool.c (revision 27838) @@ -0,0 +1,1075 @@ +/* + * stacktool.c + * + * Copyright (C) 2007-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "stacktool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool tosumMode(pxConfig *config); +static bool addsumskyfileMode(pxConfig *config); +static bool sumskyfileMode(pxConfig *config); +static bool revertsumskyfileMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupskyfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool updatesumskyfileMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); + +static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = stacktoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(STACKTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(STACKTOOL_MODE_INPUTS, inputsMode); + MODECASE(STACKTOOL_MODE_TODO, todoMode); + MODECASE(STACKTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(STACKTOOL_MODE_RESULT, resultMode); + MODECASE(STACKTOOL_MODE_REVERT, revertMode); + MODECASE(STACKTOOL_MODE_UPDATERESULT, updateresultMode); + MODECASE(STACKTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(STACKTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *having = psMetadataAlloc(); // HAVING clause + + PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "stackRun.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-select_filter", "stackRun.filter", "LIKE"); + + // these are used to build the HAVING restriction + PXOPT_COPY_S32(config->args, having, "-min_num", "num", ">="); + PXOPT_COPY_S32(config->args, having, "-min_filter", "num_filter", ">="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(where); + psFree(having); + return false; + } + + psString select = pxDataGet("staticskytool_definebyquery_select.sql"); + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + psFree(having); + return false; + } + + psString where1 = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&where1, "\nAND %s", whereClause); + psFree(whereClause); + } + + psString where2 = psStringCopy(""); + if (label) { + psStringAppend(&where2, "\nWHERE stackRun.label = '%s'", label); + } + + // Restriction on aggregated quantities using HAVING + { + psString havingClause = NULL; // HAVING string + if (psListLength(having->list)) { + havingClause = psDBGenerateWhereConditionSQL(having, NULL); + } + + if (min_new > 0) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + psStringAppend(&havingClause, + " (num_warp - num_stack >= %d OR (num_warp >= %d AND num_stack IS NULL))", + min_new, min_new); + } + if (isfinite(min_frac)) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + // Avoiding division by zero + psStringAppend(&havingClause, " (num_warp >= %f * num_stack OR num_stack IS NULL)", + (double)min_frac); + } + if (havingClause) { + psStringAppend(&select, " HAVING %s", havingClause); + psFree(havingClause); + } + } + psFree(having); + + if (!p_psDBRunQueryF(config->dbh, select, where1, where2)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + psFree(where1); + psFree(where2); + return false; + } + psFree(select); + psFree(where1); + psFree(where2); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("stacktool: no rows found"); + psFree(output); + psFree(where); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } + + psString insert = NULL; // Insertion query + if (randomLimit > 0) { + insert = pxDataGet("stacktool_definebyquery_insert_random_part1.sql"); + } else { + insert = pxDataGet("stacktool_definebyquery_insert.sql"); + } + if (!insert) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&insert, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (randomLimit > 0) { + psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(insert); + return false; + } + psStringAppend(&insert, "%s", part2); + psFree(part2); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + // pull out the skycell_id, tess_id, filter + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup skycell_id"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup tess_id"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString filter = psMetadataLookupStr(&status, row, "filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup filter"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // create a new stackRun for this stack + stackRunRow *run = stackRunRowAlloc( + 0, // ID + "new", // state + workdir, + label, + data_group ? data_group : label, + dist_group, + reduction, + dvodb, + registered, + skycell_id, + tess_id, + filter, + note); + + if (!stackRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // figure out the new stack_id + psS64 stack_id = psDBLastInsertID(config->dbh); + run->stack_id = stack_id; + + psArrayAdd(list, list->n, run); + psFree(run); + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, stack_id); + psStringSubstitute(&thisInsert, idString, "@STACK_ID@"); + psFree(idString); + + // replace @FILTER@, @SKYCELL_ID@, @RANDOM_LIMIT@ + psStringSubstitute(&thisInsert, filter, "@FILTER@"); + psStringSubstitute(&thisInsert, skycell_id, "@SKYCELL_ID@"); + + if (randomLimit > 0) { + psString limString = NULL; + psStringAppend(&limString, "%d", randomLimit); + psStringSubstitute(&thisInsert, limString, "@RANDOM_LIMIT@"); + psFree(limString); + } + + // XXX this insert uses a select to generate the list of warp_ids for the stack, + // we have applied a set of criteria above (WHERE) to select the relevant warps + // this insert needs to use exactly the same restrictions (race condition is probably not critical) + // the insert below seems to only restrict matches to the skycell, tess, and filter + if (!p_psDBRunQuery(config->dbh, thisInsert)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(thisInsert); + +# if (0) + { + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("stacktool (definebyquery, insert): no rows found"); + psFree(output); + psFree(where); + return true; + } + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } +# endif + + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + if (!stackRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + +#ifdef notdef + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); +#endif + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE stackRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true); + + psFree(query); + psFree(where); + + return result; + +#ifdef notdef + // Hack-y work around to make stacktool more like the other tools, without breaking other stuff (hopefully). + + if ((state)&&(stack_id)) { + // set detRun.state to state + return setstackRunState(config, stack_id, state); + } + + if ((state)&&(label)) { + return setstackRunStateByLabel(config, label, state); + } + + psError(PS_ERR_UNKNOWN, false, "Required options not found."); + return false; +#endif +} + + +static bool inputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX require at least a stack id (add better search options) + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool todoMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_tosum.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_stack, config->args, "-dtime_stack", false, false); + PXOPT_LOOKUP_F32(dtime_match_mean, config->args, "-dtime_match_mean", false, false); + PXOPT_LOOKUP_F32(dtime_match_stdev, config->args, "-dtime_match_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_initial, config->args, "-dtime_initial", false, false); + PXOPT_LOOKUP_F32(dtime_reject, config->args, "-dtime_reject", false, false); + PXOPT_LOOKUP_F32(dtime_final, config->args, "-dtime_final", false, false); + PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_F32(match_mean, config->args, "-match_mean", false, false); + PXOPT_LOOKUP_F32(match_stdev, config->args, "-match_stdev", false, false); + PXOPT_LOOKUP_F32(match_rms, config->args, "-match_rms", false, false); + PXOPT_LOOKUP_F32(stamps_mean, config->args, "-stamps_mean", false, false); + PXOPT_LOOKUP_F32(stamps_stdev, config->args, "-stamps_stdev", false, false); + PXOPT_LOOKUP_S32(stamps_min, config->args, "-stamps_min", false, false); + PXOPT_LOOKUP_S32(reject_images, config->args, "-reject_images", false, false); + PXOPT_LOOKUP_F32(reject_pix_mean, config->args, "-reject_pix_mean", false, false); + PXOPT_LOOKUP_F32(reject_pix_stdev, config->args, "-reject_pix_stdev", false, false); + PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX need to validate the stack_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!stackSumSkyfileInsert(config->dbh, + stack_id, + uri, + path_base, + bg, + bg_stdev, + dtime_stack, + dtime_match_mean, + dtime_match_stdev, + dtime_initial, + dtime_reject, + dtime_final, + dtime_phot, + dtime_script, + match_mean, + match_stdev, + match_rms, + stamps_mean, + stamps_stdev, + stamps_min, + reject_images, + reject_pix_mean, + reject_pix_stdev, + sources, + hostname, + good_frac, + fault, + quality + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (fault == 0) { + if (!setstackRunState(config, stack_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool resultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); + +// The following three selectors are incompatible with the sql so omit them +// PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); +// PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); +// PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_sumskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + + return true; +} + + +static bool updateresultState(pxConfig *config, psS64 stack_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, stack_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for stack_id %"PRId64, stack_id); + return false; + } + + return true; +} + +static bool updateresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 3; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + ExportTable tables [] = { + {"stackRun", "stacktool_export_run.sql"}, + {"stackInputSkyfile", "stacktool_export_input_skyfile.sql"}, + {"stackSumSkyfile", "stacktool_export_sum_skyfile.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "stackRun")) { + if (!pxSetStateCleaned("stackRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + int numImportTables = 2; + + char tables[2] [80] = {"stackInputSkyfile", "stackSumSkyfile"}; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + +#ifdef notdef + fprintf (stderr, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + psMetadataItem *item = psMetadataLookup (input, "stackRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + stackRunRow *stackRun = stackRunObjectFromMetadata (entry->data.md); + stackRunInsertObject (config->dbh, stackRun); + + // fprintf (stdout, "---- stack run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + for (int i = 0; i < numImportTables; i++) { + psMetadataItem *item = psMetadataLookup (input, tables[i]); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + switch (i) { + case 0: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + stackInputSkyfileRow *stackInputSkyfile = stackInputSkyfileObjectFromMetadata (entry->data.md); + stackInputSkyfileInsertObject (config->dbh, stackInputSkyfile); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + + case 1: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + stackSumSkyfileRow *stackSumSkyfile = stackSumSkyfileObjectFromMetadata (entry->data.md); + stackSumSkyfileInsertObject (config->dbh, stackSumSkyfile); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + } + } + + return true; +} Index: branches/tap_branches/ippTools/src/staticskytool.h =================================================================== --- branches/tap_branches/ippTools/src/staticskytool.h (revision 27838) +++ branches/tap_branches/ippTools/src/staticskytool.h (revision 27838) @@ -0,0 +1,41 @@ +/* + * staticskytool.h + * + * Copyright (C) 2007,2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef STATICSKYTOOL_H +#define STATICSKYTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + STATICSKYTOOL_MODE_NONE = 0x0, + STATICSKYTOOL_MODE_DEFINEBYQUERY, + STATICSKYTOOL_MODE_UPDATERUN, + STATICSKYTOOL_MODE_INPUTS, + STATICSKYTOOL_MODE_TODO, + STATICSKYTOOL_MODE_ADDRESULT, + STATICSKYTOOL_MODE_RESULT, + STATICSKYTOOL_MODE_REVERT, + STATICSKYTOOL_MODE_UPDATERESULT, + STATICSKYTOOL_MODE_EXPORTRUN, + STATICSKYTOOL_MODE_IMPORTRUN +} staticskytoolMode; + +pxConfig *staticskytoolConfig(pxConfig *config, int argc, char **argv); + +#endif // STATICSKYTOOL_H Index: branches/tap_branches/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/staticskytoolConfig.c (revision 27838) +++ branches/tap_branches/ippTools/src/staticskytoolConfig.c (revision 27838) @@ -0,0 +1,173 @@ +/* + * staticskytoolConfig.c + * + * Copyright (C) 2007-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "staticskytool.h" + +pxConfig *staticskytoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ss_id", 0, "search by stack ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -inputs + psMetadata *inputsArgs = psMetadataAlloc(); + psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -todo + psMetadata *todoArgs = psMetadataAlloc(); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(todoArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresult + psMetadata *addresultArgs = psMetadataAlloc(); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ss_id", 0, "define stack ID (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS32(addresultArgs, PS_LIST_TAIL, "-sources", 0, "number of sources", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-good_frac", 0, "define fraction of good pixels", NAN); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0) + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -result + psMetadata *resultArgs= psMetadataAlloc(); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revert + psMetadata *revertArgs= psMetadataAlloc(); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); + + // -updateresult + psMetadata *updateresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ss_id", 0, "define staticksky ID (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ss_id", 0, "export this staticsky ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "Define a new run", STATICSKYTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "Update a run", STATICSKYTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-inputs", "Get inputs", STATICSKYTOOL_MODE_INPUTS, inputsArgs); + PXOPT_ADD_MODE("-todo", "Get runs to do", STATICSKYTOOL_MODE_TODO, todoArgs); + PXOPT_ADD_MODE("-addresult", "Add result of run", STATICSKYTOOL_MODE_ADDRESULT, addresultsArgs); + PXOPT_ADD_MODE("-result", "Get result of run", STATICSKYTOOL_MODE_RESULT, resultArgs); + PXOPT_ADD_MODE("-revert", "Revert failed run", STATICSKYTOOL_MODE_REVERT, revertArgs); + PXOPT_ADD_MODE("-updateresult", "Update fault for run", STATICSKYTOOL_MODE_UPDATERESULT, updateresultArgs); + PXOPT_ADD_MODE("-exportrun", "Export run", STATICSKYTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "Import run", STATICSKYTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/tap_branches/ippTools/src/warptool.c =================================================================== --- branches/tap_branches/ippTools/src/warptool.c (revision 25900) +++ branches/tap_branches/ippTools/src/warptool.c (revision 27838) @@ -59,4 +59,6 @@ static bool importrunMode(pxConfig *config); static bool runstateMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); @@ -104,5 +106,5 @@ MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); - MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); @@ -110,4 +112,6 @@ MODECASE(WARPTOOL_MODE_IMPORTRUN, importrunMode); MODECASE(WARPTOOL_MODE_RUNSTATE, runstateMode); + MODECASE(WARPTOOL_MODE_LISTRUN, listrunMode); + MODECASE(WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); default: @@ -247,10 +251,10 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); // define using fake label if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -279,5 +283,5 @@ psString query = pxDataGet("warptool_definebyquery.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -383,19 +387,27 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "warpRun.reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group","warpRun.data_group", "=="); - PXOPT_COPY_STR(config->args, where, "-dist_group","warpRun.dist_group", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group","warpRun.data_group","=="); + PXOPT_COPY_STR(config->args, where, "-dist_group","warpRun.dist_group","=="); + pxAddLabelSearchArgs(config, where, "-label", "warpRun.label", "=="); + + PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpRun.registered", ">="); + PXOPT_COPY_TIME(config->args, where, "-registered_end", "warpRun.registered", "<"); + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + if (destreaked) { + psMetadataAddS64(where, PS_LIST_TAIL, "warpRun.magicked", PS_META_DUPLICATE_OK, ">", 0); + } if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE warpRun JOIN warpSkyfile USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "warpRun", true); + bool result = pxUpdateRun(config, where, &query, "warpRun", "warp_id", "warpSkyfile", true); psFree(query); @@ -419,5 +431,5 @@ psString query = pxDataGet("warptool_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -493,5 +505,5 @@ psString query = pxDataGet("warptool_imfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -567,5 +579,5 @@ psString query = pxDataGet("warptool_tooverlap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -676,5 +688,4 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyCellMap.tess_id", "=="); - // PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyCellMap.fault", "=="); @@ -683,5 +694,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -691,5 +702,5 @@ psString query = pxDataGet("warptool_revertoverlap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -823,5 +834,5 @@ psString query = pxDataGet("warptool_scmap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -896,11 +907,12 @@ psString query = pxDataGet("warptool_towarped.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereStr, "\n AND %s", whereClause); psFree(whereClause); } @@ -908,15 +920,29 @@ // treat limit == 0 as "no limit" + psString limitString = psStringCopy("\n"); if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); + // We apply the limit to both sides of the UNION + // to avoid slow queries and to the query itself + // to satisfy the user's requested limit + psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + } + +#define TWO_WHERES +#ifdef TWO_WHERES + if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr, limitString)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); return false; } +#else + if (!p_psDBRunQueryF(config->dbh, query, whereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } +#endif + psFree(limitString); + psFree(whereStr); psFree(query); @@ -977,5 +1003,5 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); // default values @@ -1040,5 +1066,5 @@ psString query = pxDataGet("warptool_finished_run_select.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); return false; } @@ -1116,5 +1142,5 @@ psString query = pxDataGet("warptool_finished_run_select.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); return false; } @@ -1194,7 +1220,8 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -1206,5 +1233,5 @@ psString query = pxDataGet("warptool_warped.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1214,9 +1241,9 @@ psString where2 = NULL; if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) { - psError(PXTOOLS_ERR_DATA, false, "pxMagicAddWhere failed"); + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); return false; } if (!pxspaceAddWhere(config, &where2, "rawExp")) { - psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); return false; } @@ -1224,15 +1251,19 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); } else if (!all && !where2) { - psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); - return false; + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } } psFree(where); - - if (where2) { - psStringAppend(&query, " %s", where2); - } // treat limit == 0 as "no limit" @@ -1294,60 +1325,56 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); - // PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Delete product - int numDeleted; // Number deleted - { - psString query = pxDataGet("warptool_revertwarped_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("warptool_revertwarped_delete.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString query_updated = pxDataGet("warptool_revertwarped_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkyfiles", numDeleted); - psFree(where); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + // fix any faulted warpSkyfiles in data_state 'update' + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("warptool", PS_LOG_INFO, "Updated %d warpSkyfiles", numUpdated); return true; @@ -1436,5 +1463,5 @@ psString query = pxDataGet("warptool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1499,5 +1526,5 @@ psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1560,5 +1587,5 @@ psString query = pxDataGet("warptool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1648,7 +1675,20 @@ } - // note only updates if warpRun.state = run_state - // XXX note that we have removed this constraint for now - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) { + psString set_magicked_skyfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (!strcmp(data_state, "full")) { + // magicked is only an argument for for -tofullskyfile + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + if (magicked) { + psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked); + psStringAppend(&set_magicked_run, "\n, warpRun.magicked = %" PRId64, magicked); + } + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + // if magicked is currently nonzero set it to -1 + // Set warpRun.magicked when the first skyfile is cleaned + psStringAppend(&set_magicked_skyfile, "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1), warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)"); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); // rollback @@ -1660,7 +1700,8 @@ } psFree(query); + psFree(set_magicked_skyfile); query = pxDataGet("warptool_change_run_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) { // rollback if (!psDBRollback(config->dbh)) { @@ -1670,4 +1711,5 @@ return false; } + psFree(set_magicked_run); if (!psDBCommit(config->dbh)) { @@ -1707,5 +1749,5 @@ if (!state) { psString query = pxDataGet("warptool_updateskyfile.sql"); - + if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1716,15 +1758,15 @@ else { if (strcmp(state,"error_cleaned") == 0) { - change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); + change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); } else if (strcmp(state, "error_scrubbed") == 0) { - change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); + change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); } else if (strcmp(state, "error_purged") == 0) { - change_skyfile_data_state(config,"error_purged","goto_purged"); + change_skyfile_data_state(config,"error_purged","goto_purged"); } else { - psError(PS_ERR_UNKNOWN, false, "unhandled state given"); - return(false); + psError(PS_ERR_UNKNOWN, false, "unhandled state given"); + return(false); } } @@ -1772,5 +1814,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1939,5 +1981,5 @@ psString query = pxDataGet("warptool_runstate.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1948,5 +1990,5 @@ psFree(whereClause); } else { - psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); return false; } @@ -2000,2 +2042,154 @@ return true; } + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpRun.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "warpRun.dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpRun")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, warp_id DESC"); + } + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("warptool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , warpRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} Index: branches/tap_branches/ippTools/src/warptool.h =================================================================== --- branches/tap_branches/ippTools/src/warptool.h (revision 25900) +++ branches/tap_branches/ippTools/src/warptool.h (revision 27838) @@ -55,4 +55,6 @@ WARPTOOL_MODE_IMPORTRUN, WARPTOOL_MODE_RUNSTATE, + WARPTOOL_MODE_LISTRUN, + WARPTOOL_MODE_SETSKYFILETOUPDATE, } warptoolMode; Index: branches/tap_branches/ippTools/src/warptoolConfig.c =================================================================== --- branches/tap_branches/ippTools/src/warptoolConfig.c (revision 25900) +++ branches/tap_branches/ippTools/src/warptoolConfig.c (revision 27838) @@ -40,5 +40,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -133,7 +133,11 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by warpRun reduction class", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by warpRun state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by warpRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by warpRun data_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by warpRun dist_group", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_begin", 0, "search for runs by registration time (>=)", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_end", 0, "search for runs by registration time (<)", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); @@ -142,4 +146,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + // -exp psMetadata *expArgs = psMetadataAlloc(); @@ -212,5 +217,5 @@ psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); - psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked", 0, "define if this skycell has been magicked", false); + psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-set_magicked", 0, "define if this skycell has been magicked", 0); psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); @@ -230,4 +235,6 @@ psMetadataAddS16(warpedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL); // add magic related arguments pxmagicAddArguments(warpedArgs); @@ -238,4 +245,30 @@ psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); + psMetadataAddS16(listrunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL); + // add magic related arguments + pxmagicAddArguments(listrunArgs); + // add arguments for spatial search + pxspaceAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertwarped @@ -309,6 +342,7 @@ psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); - - // -toupdateskyfile + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-set_magicked", 0, "new value for magicked", 0); + + // -updateskyfile psMetadata *updateskyfileArgs = psMetadataAlloc(); psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); @@ -316,5 +350,5 @@ psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-fault", 0, "new fault code", 0); psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", 0); - + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -339,4 +373,10 @@ psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -setskyfiletoupdate + psMetadata *setskyfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setskyfiletoupdateArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID (required)", 0); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "new value for warpRun.label", NULL); @@ -358,4 +398,5 @@ PXOPT_ADD_MODE("-advancerun", "", WARPTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-listrun", "", WARPTOOL_MODE_LISTRUN, listrunArgs); PXOPT_ADD_MODE("-runstate", "", WARPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); @@ -371,4 +412,5 @@ PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); + PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", WARPTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", WARPTOOL_MODE_IMPORTRUN, importrunArgs);