Index: trunk/ippTools/src/dettool.c =================================================================== --- trunk/ippTools/src/dettool.c (revision 18336) +++ trunk/ippTools/src/dettool.c (revision 18561) @@ -18,17 +18,4 @@ */ -#ifdef HAVB_CONFIG_H -#include -#endif - -#include -#include -#include -#include -#include - -#include - -#include "pxtools.h" #include "dettool.h" @@ -98,5 +85,5 @@ //static psArray *searchInputImfiles(pxConfig *config, const char *det_id); static detInputExpRow *rawDetrenTodetInputExpRow(rawExpRow *rawExp, psS64 det_id, psS32 iteration); -static psArray *searchRawImfiles(pxConfig *config, psMetadata *where); +static psArray *searchRawImfiles(pxConfig *config); static bool startNewIteration(pxConfig *config, psS64 det_id); static psS32 incrementIteration(pxConfig *config, psS64 det_id); @@ -214,20 +201,23 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = psStringCopy( - "SELECT" - " rawExp.*" - " FROM rawExp" - " LEFT JOIN detInputExp" - " ON rawExp.exp_id = detInputExp.exp_id" - " WHERE" - " detInputExp.exp_id IS NULL" - " AND rawExp.object != 'object'" - ); - - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp"); + psString query = pxDataGet("dettool_pending.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -315,4 +305,5 @@ // make sure that -exp_id was parsed correctly // XXX this can be removed someday + // XXX Special case for a multi : move into a macro? if (item->type == PS_DATA_METADATA_MULTI) { psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); @@ -472,5 +463,5 @@ #if 0 // This function is used to convert the det_id from an int, as it is used -// internally, to be a string for external use. The rational being that we may +// internally, to be a string for external use. The rationale being that we may // want to change how det_id is generated in the future and don't want to // external programs to become depending on this value being an int. @@ -543,5 +534,4 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-select_exp_type", "exp_type", "=="); - // map -inst -> camera PXOPT_COPY_STR(config->args, where, "-select_inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-select_telescope", "telescope", "=="); @@ -555,12 +545,12 @@ PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "exp_time", ">="); PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "exp_time", "<="); - PXOPT_COPY_F32(config->args, where, "-select_ccd_temp_min", "ccd_temp", ">="); - PXOPT_COPY_F32(config->args, where, "-select_ccd_temp_max", "ccd_temp", "<="); - PXOPT_COPY_F32(config->args, where, "-select_pon_time_min", "pon_time", ">="); - PXOPT_COPY_F32(config->args, where, "-select_pon_time_max", "pon_time", "<="); - PXOPT_COPY_F32(config->args, where, "-select_posang_min", "posang", ">="); - PXOPT_COPY_F32(config->args, where, "-select_posang_max", "posang", "<="); - PXOPT_COPY_F32(config->args, where, "-select_solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-select_solang_max", "solang", "<="); + PXOPT_COPY_F64(config->args, where, "-select_ccd_temp_min", "ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-select_ccd_temp_max", "ccd_temp", "<="); + PXOPT_COPY_F64(config->args, where, "-select_pon_time_min", "pon_time", ">="); + PXOPT_COPY_F64(config->args, where, "-select_pon_time_max", "pon_time", "<="); + PXOPT_COPY_F64(config->args, where, "-select_posang_min", "posang", ">="); + PXOPT_COPY_F64(config->args, where, "-select_posang_max", "posang", "<="); + PXOPT_COPY_F64(config->args, where, "-select_solang_min", "solang", ">="); + PXOPT_COPY_F64(config->args, where, "-select_solang_max", "solang", "<="); if (!psListLength(where->list)) { @@ -571,5 +561,5 @@ // there is some namespace overlap between the names of the fields we'd // like to search by to setup a detrun and the names of the fields we'd - // like to assign values to so I've seperated them but prepending set- to + // like to assign values to so I've separated them but prepending set- to // the assigned values @@ -731,34 +721,22 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); - - // optional - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_STR(det_type, config->args, "-set_det_type", false, false); - PXOPT_LOOKUP_STR(mode, config->args, "-set_mode", false, false); - // check mode - if (mode && !isValidMode(config, mode)) { - psError(PS_ERR_UNKNOWN, false, "invalud mode"); - return false; - } - PXOPT_LOOKUP_STR(camera, config->args, "-set_inst", false, false); - PXOPT_LOOKUP_STR(telescope, config->args, "-set_telescope", false, false); + + PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_STR(det_type, config->args, "-set_det_type", false, false); // optional + PXOPT_LOOKUP_STR(mode, config->args, "-set_mode", false, false); // optional PXOPT_LOOKUP_STR(exp_type, config->args, "-set_exp_type", false, false); PXOPT_LOOKUP_STR(filelevel, config->args, "-set_filelevel", false, false); PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(filter, config->args, "-set_filter", false, false); - PXOPT_LOOKUP_F32(airmass_min, config->args, "-set_airmass_min", false, false); PXOPT_LOOKUP_F32(airmass_max, config->args, "-set_airmass_max", false, false); PXOPT_LOOKUP_F32(exp_time_min, config->args, "-set_exp_time_min", false, false); PXOPT_LOOKUP_F32(exp_time_max, config->args, "-set_exp_time_max", false, false); - PXOPT_LOOKUP_F32(ccd_temp_min, config->args, "-set_ccd_temp_min", false, false); - PXOPT_LOOKUP_F32(ccd_temp_max, config->args, "-set_ccd_temp_max", false, false); - PXOPT_LOOKUP_F32(posang_min, config->args, "-set_posang_min", false, false); - PXOPT_LOOKUP_F32(posang_max, config->args, "-set_posang_max", false, false); - PXOPT_LOOKUP_F32(solang_min, config->args, "-set_solang_min", false, false); - PXOPT_LOOKUP_F32(solang_max, config->args, "-set_solang_max", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_F64(ccd_temp_min, config->args, "-set_ccd_temp_min", false, false); + PXOPT_LOOKUP_F64(ccd_temp_max, config->args, "-set_ccd_temp_max", false, false); + PXOPT_LOOKUP_F64(posang_min, config->args, "-set_posang_min", false, false); + PXOPT_LOOKUP_F64(posang_max, config->args, "-set_posang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-set_solang_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-set_solang_max", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); PXOPT_LOOKUP_TIME(time_begin, config->args, "-set_time_begin", false, false); @@ -767,5 +745,12 @@ PXOPT_LOOKUP_TIME(use_end, config->args, "-use_end", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // optional + + // check mode + if (mode && !isValidMode(config, mode)) { + psError(PS_ERR_UNKNOWN, false, "invalud mode"); + return false; + } // lookup the detRun that we will be basing this one on @@ -814,14 +799,4 @@ } - if (camera) { - psFree(detRun->camera); - detRun->camera = psStringCopy(camera); - } - - if (telescope) { - psFree(detRun->telescope); - detRun->telescope = psStringCopy(telescope); - } - if (exp_type) { psFree(detRun->exp_type); @@ -925,6 +900,6 @@ // XXX: possible mem leak: PXOPT_COPY* will free time_filter but NOTHING // that has previously been allocated - PXOPT_COPY_TIME(config->args, time_filter, "-filter_input_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, time_filter, "-filter_input_end", "dateobs", "<"); + PXOPT_COPY_TIME(config->args, time_filter, "-set_input_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, time_filter, "-set_input_end", "dateobs", "<"); // start a transaction so we don't end up with childlessed det_ids @@ -951,16 +926,9 @@ psS64 newDet_id = psDBLastInsertID(config->dbh); - psString query = psStringCopy( - "INSERT INTO detInputExp" - " SELECT" - " %d," - " 0," - " detResidExp.exp_id," - " detResidExp.accept" - " FROM detResidExp" - " JOIN rawExp" - " USING(exp_id)" - " WHERE det_id = %d" - ); + psString query = pxDataGet("dettool_definebydetrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } if (time_filter->list->n) { @@ -1017,27 +985,31 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + + // airmass_min < airmass < airmass_max + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_min", "<="); + PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_max", ">="); + + // exp_time_min < exp_time < exp_time_max + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_min", "<="); + PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_max", ">="); + + // ccd_temp_min < ccd_temp < ccd_temp_max + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">="); + + PXOPT_COPY_F64(config->args, where, "-posang", "posang_min", "<="); + PXOPT_COPY_F64(config->args, where, "-posang", "posang_max", ">="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(active, config->args, "-active", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_COPY_STR(config->args, config->where, "-inst", "camera", "=="); - PXOPT_COPY_STR(config->args, config->where, "-telescope", "telescope", "=="); - PXOPT_COPY_STR(config->args, config->where, "-filter", "filter", "=="); - - // airmass_min < airmass < airmass_max - PXOPT_COPY_F32(config->args, config->where, "-airmass", "airmass_min", "<="); - PXOPT_COPY_F32(config->args, config->where, "-airmass", "airmass_max", ">="); - - // exp_time_min < exp_time < exp_time_max - PXOPT_COPY_F32(config->args, config->where, "-exp_time", "exp_time_min", "<="); - PXOPT_COPY_F32(config->args, config->where, "-exp_time", "exp_time_max", ">="); - - // ccd_temp_min < ccd_temp < ccd_temp_max - PXOPT_COPY_F32(config->args, config->where, "-ccd_temp", "ccd_temp_min", "<="); - PXOPT_COPY_F32(config->args, config->where, "-ccd_temp", "ccd_temp_max", ">="); - - PXOPT_COPY_F64(config->args, config->where, "-posang", "posang_min", "<="); - PXOPT_COPY_F64(config->args, config->where, "-posang", "posang_max", ">="); - psString query = pxDataGet("dettool_runs.sql"); if (!query) { @@ -1046,17 +1018,18 @@ } - if (config->where && active) { - psString whereClause = psDBGenerateWhereSQL(config->where, "detRun"); + if (psListLength(where->list) && active) { + psString whereClause = psDBGenerateWhereSQL(where, "detRun"); psStringAppend(&query, " %s AND (detRun.state = 'run' OR detRun.state = 'stop' OR detRun.state = 'register')", whereClause); psFree(whereClause); } - if (config->where && !active) { - psString whereClause = psDBGenerateWhereSQL(config->where, "detRun"); + if (psListLength(where->list) && !active) { + psString whereClause = psDBGenerateWhereSQL(where, "detRun"); psStringAppend(&query, " %s", whereClause); psFree(whereClause); } - if (!config->where && active) { + if (!psListLength(where->list) && active) { psStringAppend(&query, " WHERE (detRun.state = 'run' OR detRun.state = 'stop' OR detRun.state = 'register')"); } + psFree (where); // treat limit == 0 as "no limit" @@ -1080,14 +1053,4 @@ } -// XXX old version -# if 0 - // XXX fix the hard coding of the table name - psArray *runs = psDBSelectRows(config->dbh, "detRun", config->where, 0); - if (!runs) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } -# endif - if (!psArrayLength(runs)) { psTrace("dettool", PS_LOG_INFO, "no rows found"); @@ -1112,24 +1075,24 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = psStringCopy( - "SELECT DISTINCT\n" - " detRun.*\n" - " FROM detRun\n" - " LEFT JOIN detRun as foo\n" - " ON foo.parent = detRun.det_id\n" - " WHERE\n" - " detRun.state = 'stop'\n" - " AND detRun.mode = 'master'\n" - " AND foo.det_id IS NULL\n" - ); - - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detRun"); + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("dettool_childlessrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRun"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree (where); // treat limit == 0 as "no limit" @@ -1188,21 +1151,22 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // select detInputExp.* - // select rawExp.* - // by: - // exp_id - - psString query = psStringCopy( - "SELECT DISTINCT *" - " FROM detInputExp" - " JOIN rawExp" - " USING(exp_id)" - ); - - if (config->where) { - psString whereClause = psDBGenerateWhereSQL(config->where, "detInputExp"); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("dettool_input.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, "detInputExp"); psStringAppend(&query, " %s", whereClause); psFree(whereClause); } + psFree (where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -1242,4 +1206,6 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + psMetadata *where = psMetadataAlloc(); + psString query = pxDataGet("dettool_raw.sql"); if (!query) { @@ -1248,9 +1214,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree (where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -1288,4 +1255,9 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -1297,9 +1269,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree (where); // treat limit == 0 as "no limit" @@ -1340,55 +1313,9 @@ } -static psArray *searchRawImfiles(pxConfig *config, psMetadata *where) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - // use the default where if "where" is NULL - if (!where) { - where = config->where; - } - - // select exp_ids from detInputExp matching det_idp - // where query should be pre-generated - psArray *detInputExp = - detInputExpSelectRowObjects(config->dbh, where, 0); - if (!detInputExp) { - psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); - return NULL; - } - - // generate where query with just the exp_ids - psMetadata *where_exp_ids = psMetadataAlloc(); - for (long i = 0; i < psArrayLength(detInputExp); i++) { - detInputExpRow *row = detInputExp->data[i]; - if (!psMetadataAddS64(where_exp_ids, PS_LIST_TAIL, "exp_id", - PS_META_DUPLICATE_OK, "==", row->exp_id) - ) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(detInputExp); - psFree(where_exp_ids); - return NULL; - } - } - psFree(detInputExp); - - // select rawImfiles with matching exp_ids - psArray *rawImfiles = - rawImfileSelectRowObjects(config->dbh, where_exp_ids, 0); - psFree(where_exp_ids); - if (!rawImfiles) { - psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); - return NULL; - } - - return rawImfiles; -} - static bool addprocessedimfileMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - // det_id, exp_id, class_id, uri, recipe, -bg, -bg_stdev - // are required + // det_id, exp_id, class_id, uri, recipe, -bg, -bg_stdev are required PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); @@ -1416,25 +1343,16 @@ PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - // find the matching rawImfile by exp_id/class_id psMetadata *where = psMetadataAlloc(); - if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item class_id"); - psFree(where); - return false; - } + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); psArray *rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); psFree(where); + if (!rawImfiles) { psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found "); return false; } - // create a new detProcessedImfile object @@ -1473,4 +1391,138 @@ } +static bool processedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + char *value = NULL; + + PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("dettool_processedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // add the two required restrictions: detRun.state and detRun.mode + // NOTE the above query requires one of the following two WHERE statements + { + bool status; + if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { + psStringAppend(&query, " WHERE detRun.state = '%s'", value); + } else { + psStringAppend(&query, " WHERE detRun.state = 'run'"); + } + if ((value = psMetadataLookupStr(&status, config->args, "-select_mode"))) { + psStringAppend(&query, " AND detRun.mode = '%s'", value); + } else { + psStringAppend(&query, " AND detRun.mode = 'master'"); + } + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree (where); + + // restrict search to included imfiles + if (included) { + psStringAppend(&query, " AND detInputExp.include = 1"); + } + + if (faulted) { + // list only faulted rows + psStringAppend(&query, " %s", "AND detProcessedImfile.fault != 0"); + } else { + // don't list faulted rows + psStringAppend(&query, " %s", "AND detProcessedImfile.fault = 0"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + + psString query = pxDataGet("dettool_revertprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detProcessedImfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + return true; +} + static bool toprocessedexpMode(pxConfig *config) { @@ -1485,13 +1537,4 @@ return false; } - - // XXX does it make sense to accept any search params? -#if 0 - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } -#endif // treat limit == 0 as "no limit" @@ -1561,41 +1604,9 @@ PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); - - psString query = psStringCopy( - " SELECT DISTINCT" - " detProcessedImfile.det_id," - " detRun.iteration," - " detRun.det_type," - " detProcessedImfile.exp_id" - " FROM detRun" - " JOIN detInputExp" - " ON detRun.det_id = detInputExp.det_id" - " AND detRun.iteration = detInputExp.iteration" - " JOIN rawExp" - " ON detInputExp.exp_id = rawExp.exp_id" - " JOIN detProcessedImfile" - " ON detInputExp.det_id = detProcessedImfile.det_id" - " AND detInputExp.exp_id = detProcessedImfile.exp_id" - " LEFT JOIN detProcessedExp" - " ON detInputExp.det_id = detProcessedExp.det_id" - " AND detProcessedImfile.exp_id= detProcessedExp.exp_id" - " LEFT JOIN rawImfile" - " ON detInputExp.exp_id = rawImfile.exp_id" - " AND detProcessedImfile.class_id = rawImfile.class_id" - " WHERE" - " detRun.state = 'run'" - " AND detRun.mode = 'master'" - " AND detProcessedExp.det_id IS NULL" - " AND detProcessedExp.exp_id IS NULL" - " AND detInputExp.include = 1" - " AND detRun.det_id = %s" - " AND detProcessedImfile.exp_id = '%s'" - " GROUP BY" - " detProcessedImfile.class_id," - " rawImfile.class_id," - " detRun.det_id" - " HAVING" - " COUNT(detProcessedImfile.class_id) = COUNT(rawImfile.class_id)" - ); + psString query = pxDataGet("dettool_addprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } if (!p_psDBRunQuery(config->dbh, query, det_id, exp_id)) { @@ -1654,15 +1665,20 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); psString query = psStringCopy("SELECT * FROM detProcessedExp"); - if (config->where) { - psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); psStringAppend(&query, " %s", whereClause); psFree(whereClause); } + psFree(where); if (faulted) { @@ -1716,4 +1732,9 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + psString query = pxDataGet("dettool_revertprocessedexp.sql"); if (!query) { @@ -1722,9 +1743,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedExp"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detProcessedExp"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -1756,13 +1778,4 @@ return false; } - - // XXX does it make sense to accept any search params? -#if 0 - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } -#endif // treat limit == 0 as "no limit" @@ -1803,143 +1816,49 @@ } -static bool processedimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - char *value = NULL; - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - - psString query = psStringCopy( - " SELECT DISTINCT" - " detRun.det_type," - " rawExp.exp_time," - " detProcessedImfile.*" - " FROM detProcessedImfile" - " JOIN detRun" - " USING(det_id)" - " JOIN detInputExp" - " ON detRun.det_id = detInputExp.det_id" - " AND detRun.iteration = detInputExp.iteration" - " AND detProcessedImfile.exp_id = detInputExp.exp_id" - " JOIN rawExp" - " ON rawExp.exp_id = detProcessedImfile.exp_id" - " WHERE" - ); - // NOTE the above WHERE is completed with the following line: - - // add the two required restrictions: detRun.state and detRun.mode - { - bool status; - if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { - psStringAppend(&query, " detRun.state = '%s'", value); - } else { - psStringAppend(&query, " detRun.state = 'run'"); +// this is NOT a command-line mode : it is used by 'addstackedMode' +static psArray *searchRawImfiles(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + // select exp_ids from detInputExp matching det_id & iteration + // where query should be pre-generated + psArray *detInputExp = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree (where); + + if (!detInputExp) { + psError(PS_ERR_UNKNOWN, false, "no rawExp rows found"); + return NULL; + } + + // generate where query with just the exp_ids + psMetadata *where_exp_ids = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(detInputExp); i++) { + detInputExpRow *row = detInputExp->data[i]; + if (!psMetadataAddS64(where_exp_ids, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", row->exp_id) + ) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(detInputExp); + psFree(where_exp_ids); + return NULL; } - if ((value = psMetadataLookupStr(&status, config->args, "-select_mode"))) { - psStringAppend(&query, " AND detRun.mode = '%s'", value); - } else { - psStringAppend(&query, " AND detRun.mode = 'master'"); - } - } - - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedImfile"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - { - bool status = false; - bool included = psMetadataLookupBool(&status, config->args, "-included"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple"); - return false; - } - // restrict search to included imfiles - if (included) { - psStringAppend(&query, " AND detInputExp.include = 1"); - } - } - - if (faulted) { - // list only faulted rows - psStringAppend(&query, " %s", "AND detProcessedImfile.fault != 0"); - } else { - // don't list faulted rows - psStringAppend(&query, " %s", "AND detProcessedImfile.fault = 0"); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("dettool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - - -static bool revertprocessedimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psString query = pxDataGet("dettool_revertprocessedimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detProcessedImfile"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - return false; - } - - return true; -} - + } + psFree(detInputExp); + + // select rawImfiles with matching exp_ids + psArray *rawImfiles = + rawImfileSelectRowObjects(config->dbh, where_exp_ids, 0); + psFree(where_exp_ids); + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} static bool addstackedMode(pxConfig *config) @@ -1947,6 +1866,7 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // det_id, class_id, uri, & recipe are required + // det_id, iteration, class_id, uri, & recipe are required PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); @@ -1967,21 +1887,7 @@ PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); - // default values - PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); - // correlate the class_id against the input exposure(s) - - // we have to generate our own where clause as we want to search only by the - // det_id - psMetadata *where = psMetadataAlloc(); - if (!psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", - (psS64)atoll(det_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - - psArray *rawImfiles = searchRawImfiles(config, where); - psFree(where); + // searchRawImfiles defines a where clause to search by det_id and iteration + psArray *rawImfiles = searchRawImfiles(config); if (!rawImfiles) { @@ -2045,28 +1951,27 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - - // select detStackedImfile.* - // by: - // where det_id, iteration, class_id is not in detNormalizedImfile - - psString query = psStringCopy( - "SELECT" - " detStackedImfile.*" - " FROM detStackedImfile" - " JOIN detRun" - " USING(det_id, iteration)" - " WHERE" - " detRun.state = 'run'" - " AND detRun.mode = 'master'" - ); - - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + + psString query = pxDataGet("dettool_stacked.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detStackedImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (faulted) { @@ -2120,4 +2025,10 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + psString query = pxDataGet("dettool_revertstacked.sql"); if (!query) { @@ -2126,9 +2037,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detStackedImfile"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detStackedImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -2159,13 +2071,4 @@ return false; } - - // XXX does it make sense to accept any search params? -#if 0 - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } -#endif // treat limit == 0 as "no limit" @@ -2240,7 +2143,12 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); psString query = pxDataGet("dettool_normalizedstat.sql"); @@ -2258,9 +2166,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); // treat limit == 0 as "no limit" @@ -2305,4 +2214,10 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + psString query = pxDataGet("dettool_revertnormalizedstat.sql"); if (!query) { @@ -2311,9 +2226,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedStatImfile"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedStatImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -2339,46 +2255,9 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // select detNormalizedStatImfile.* - // by: - // where det_id, iteration, class_id is not in detNormalizedImfile - - psString query = psStringCopy( - "SELECT DISTINCT" - " detRun.det_type," - " detRun.workdir," - " rawExp.camera," - " detStackedImfile.uri," - " detNormalizedStatImfile.*" - " FROM detRun" - " JOIN detStackedImfile" - " USING(det_id, iteration)" - " JOIN detInputExp" - " USING(det_id, iteration)" - " JOIN rawExp" - " ON detInputExp.exp_id = rawExp.exp_id" - " JOIN detNormalizedStatImfile" - " ON detStackedImfile.det_id = detNormalizedStatImfile.det_id" - " AND detStackedImfile.iteration = detNormalizedStatImfile.iteration" - " AND detStackedImfile.class_id = detNormalizedStatImfile.class_id" - " LEFT JOIN detNormalizedImfile" - " ON detNormalizedStatImfile.det_id = detNormalizedImfile.det_id" - " AND detNormalizedStatImfile.iteration = detNormalizedImfile.iteration" - " AND detNormalizedStatImfile.class_id = detNormalizedImfile.class_id" - " WHERE" - " detNormalizedImfile.det_id IS NULL" - " AND detNormalizedImfile.iteration IS NULL" - " AND detNormalizedImfile.class_id IS NULL" - " AND detNormalizedStatImfile.fault = 0" - " AND detRun.state = 'run'" - ); - - // XXX does it make sense to accept any search params? -#if 0 - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } -#endif + psString query = pxDataGet("dettool_tonormalize.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } // treat limit == 0 as "no limit" @@ -2419,95 +2298,4 @@ } -#if 0 -// XXX this function was left in commented as this method may be useful in the -// future -static psArray *validDetInputClassIds(pxConfig *config, const char *det_id) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - // det_id is input as a string because the fact that it is an integer - // is just a database impliementation detail. - PS_ASSERT_PTR_NON_NULL(det_id, NULL); - - psArray *rawImfiles = searchInputImfiles(config, det_id); - if (!rawImfiles) { - return NULL; - } - - psArray *valid_class_ids = NULL; - { - // All this jumping through hoops is so that we end up with unique - // values. PP thinks that making multiple passes through this array and - // deleting matched elements would end up being more expensive then a - // double sort and stagger scheme. JH thinks it would be cheaper to - // just do a unqiue sort and delete. So this is really just a cheap - // hack to avoid implimenting a unique sort function but at least it's - // stable. - psHash *hash = psHashAlloc(psArrayLength(rawImfiles)); - for (long i = 0; i < psArrayLength(rawImfiles); i++) { - psHashAdd(hash, - ((rawImfileRow *)rawImfiles->data[i])->class_id, - ((rawImfileRow *)rawImfiles->data[i])->class_id - ); - } - valid_class_ids = psHashToArray(hash); - psFree(hash); - } - psFree(rawImfiles); - - return valid_class_ids; -} - -static psArray *searchInputImfiles(pxConfig *config, const char *det_id) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - // det_id is input as a string because the fact that it is an integer - // is just a database impliementation detail. - PS_ASSERT_PTR_NON_NULL(det_id, NULL); - - psArray *inputExps = NULL; - { - psMetadata *where = psMetadataAlloc(); - if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", - (psS32)atoi(det_id))) { - psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); - psFree(where); - return NULL; - } - inputExps = detInputExpSelectRowObjects(config->dbh, where, 0); - psFree(where); - } - if (!inputExps) { - psError(PS_ERR_UNKNOWN, false, "no detInputExp rows found"); - return NULL; - } - - // find rawImfiles associated with detInputExps - psArray *rawImfiles = NULL; - { - psMetadata *where = psMetadataAlloc(); - for (long i = 0; i < psArrayLength(inputExps); i++) { - if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", - PS_META_DUPLICATE_OK, "==", - ((detInputExpRow *)inputExps->data[i])->exp_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(inputExps); - psFree(where); - return NULL; - } - } - psFree(inputExps); - rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); - // XXX this really should be sorted for uniqueness - psFree(where); - } - if (!rawImfiles) { - psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); - return NULL; - } - - return rawImfiles; -} -#endif - static bool addnormalizedimfileMode(pxConfig *config) { @@ -2516,4 +2304,5 @@ // required PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); @@ -2533,7 +2322,4 @@ PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - - // default values - PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); if (!detNormalizedImfileInsert(config->dbh, @@ -2560,8 +2346,13 @@ } - static bool normalizedimfileMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -2569,26 +2360,16 @@ PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - psString query = psStringCopy( - "SELECT" - " detNormalizedImfile.*" - " FROM detNormalizedImfile" - " JOIN detRun" - " USING(det_id, iteration)" - " WHERE" - " detRun.state = 'run'" - " AND detRun.mode = 'master'" - ); - - if (config->where) { - bool status; - int iteration = psMetadataLookupS32 (&status, config->where, "iteration"); - if (status) { - psMetadataRemoveKey (config->where, "iteration"); - psMetadataAddS32 (config->where, PS_LIST_TAIL, "iteration", 0, "==", iteration); - } - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedImfile"); + psString query = pxDataGet("dettool_normalizedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (faulted) { @@ -2642,4 +2423,10 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + psString query = pxDataGet("dettool_revertnormalizedimfile.sql"); if (!query) { @@ -2648,9 +2435,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedImfile"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -2668,5 +2456,4 @@ return true; } - static bool tonormalizedexpMode(pxConfig *config) @@ -2682,13 +2469,4 @@ return false; } - - // XXX does it make sense to accept any search params? -#if 0 - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } -#endif // treat limit == 0 as "no limit" @@ -2732,14 +2510,9 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); - + + PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - - // Required if code == 0 - PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); - - // optional + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); @@ -2752,7 +2525,4 @@ PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - // default values - PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); - psString query = pxDataGet("dettool_tonormalizedexp.sql"); if (!query) { @@ -2761,8 +2531,6 @@ } - psStringAppend(&query, - " WHERE det_id = %s" - " AND iteration = %d", det_id, iteration); - + // XXX in some other places, we put the arguments in the .sql and supply them to RunQuery + psStringAppend(&query, " WHERE det_id = %s AND iteration = %d", det_id, iteration); if (!p_psDBRunQuery(config->dbh, query)) { @@ -2813,24 +2581,25 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - psString query = psStringCopy( - "SELECT" - " detNormalizedExp.*" - " FROM detNormalizedExp" - " JOIN detRun" - " USING(det_id, iteration)" - " WHERE" - " detRun.state = 'run'" - " AND detRun.mode = 'master'" - ); - - if (config->where) { - psString whereClause = psDBGenerateWhereSQL(config->where, NULL); + psString query = pxDataGet("dettool_normalizedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); psStringAppend(&query, " %s", whereClause); psFree(whereClause); } + psFree(where); if (faulted) { @@ -2885,4 +2654,9 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + psString query = pxDataGet("dettool_revertnormalizedexp.sql"); if (!query) { @@ -2891,9 +2665,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detNormalizedExp"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detNormalizedExp"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -2925,13 +2700,4 @@ return false; } - - // XXX does it make sense to accept any search params? -#if 0 - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } -#endif // treat limit == 0 as "no limit" @@ -2977,16 +2743,12 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); - + + PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); // required + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - - // Required if code == 0 - PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); - PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); - - // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); // Required if code == 0 + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); @@ -2995,5 +2757,4 @@ PXOPT_LOOKUP_F64(bg_kurtosis, config->args, "-bg_kurtosis", false, false); PXOPT_LOOKUP_F64(bin_stdev, config->args, "-bin_stdev", false, false); - PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); @@ -3002,5 +2763,4 @@ PXOPT_LOOKUP_F64(fringe_resid_1, config->args, "-fringe_resid_1", false, false); PXOPT_LOOKUP_F64(fringe_resid_2, config->args, "-fringe_resid_2", false, false); - PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); @@ -3009,7 +2769,4 @@ PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - - // default values - PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); if (!detResidImfileInsert(config->dbh, @@ -3052,30 +2809,23 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - // select detResidImfile.* - // by: - // where det_id, iteration, exp_id is not in detResidExp; - - psString query = psStringCopy( - "SELECT" - " detRun.det_type," - " detRun.mode," - " detResidImfile.*," - " rawExp.exp_time" - " FROM detResidImfile" - " JOIN detRun" - " USING(det_id, iteration)" - " JOIN rawExp" - " USING(exp_id)" - " WHERE" - ); - // NOTE the above WHERE is completed with the following line: - + psString query = pxDataGet("dettool_residimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // NOTE: the query ends in a WHERE, add the required restriction on detRun.state char *value = NULL; - - // add the two required restrictions: detRun.state and detRun.mode bool status; if ((value = psMetadataLookupStr(&status, config->args, "-select_state"))) { @@ -3085,9 +2835,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidImfile"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (faulted) { @@ -3141,4 +2892,11 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + psString query = pxDataGet("dettool_revertresidimfile.sql"); if (!query) { @@ -3147,9 +2905,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidImfile"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidImfile"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -3169,14 +2928,13 @@ +/* + The SQL returns a list of exposures for which all component class ids have had residuals + created. This list includes the detrend id, iteration, exposure id, detrend type, and + whether the exposure was included in the stack for this iteration. +*/ + static bool toresidexpMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - - /* -which returns a list of exposures for which all component class ids -have had residuals created. This list includes the detrend id, -iteration, exposure id, detrend type, and whether the exposure was -included in the stack for this iteration. - */ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -3188,13 +2946,4 @@ return false; } - - // XXX does it make sense to accept any search params? -#if 0 - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } -#endif // treat limit == 0 as "no limit" @@ -3235,18 +2984,25 @@ } +/* + The SQL returns a list of exposures for which all component class ids have had residuals + created. This list includes the detrend id, iteration, exposure id, detrend type, and + whether the exposure was included in the stack for this iteration. +*/ + static bool addresidexpMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); - + // limit search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); + PXOPT_LOOKUP_STR(exp_id, config->args, "-exp_id", true, false); // required PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - - // Required if code == 0 - PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); - - // optional + PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0 PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); @@ -3255,5 +3011,4 @@ PXOPT_LOOKUP_F64(bg_kurtosis, config->args, "-bg_kurtosis", false, false); PXOPT_LOOKUP_F64(bin_stdev, config->args, "-bin_stdev", false, false); - PXOPT_LOOKUP_F64(fringe_0, config->args, "-fringe_0", false, false); PXOPT_LOOKUP_F64(fringe_1, config->args, "-fringe_1", false, false); @@ -3262,5 +3017,4 @@ PXOPT_LOOKUP_F64(fringe_resid_1, config->args, "-fringe_resid_1", false, false); PXOPT_LOOKUP_F64(fringe_resid_2, config->args, "-fringe_resid_2", false, false); - PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); @@ -3269,15 +3023,5 @@ PXOPT_LOOKUP_F64(user_5, config->args, "-user_5", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - - // default values - PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false); PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); - - /* -which returns a list of exposures for which all component class ids -have had residuals created. This list includes the detrend id, -iteration, exposure id, detrend type, and whether the exposure was -included in the stack for this iteration. - */ psString query = pxDataGet("dettool_toresidexp.sql"); @@ -3287,21 +3031,10 @@ } - { - // build a query to search by det_id, iteration, exp_id - psMetadata *where = psMetadataAlloc(); - - PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); - PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); - - - // there's not - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psFree(where); - if (whereClause) { - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - } - } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -3362,27 +3095,27 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-recip", "recipe", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - - psString query = psStringCopy( - "SELECT" - " detRun.mode," - " detResidExp.*," - " detInputExp.include" - " FROM detRun" - " JOIN detInputExp" - " USING(det_id, iteration)" - " JOIN detResidExp" - " USING(det_id, iteration, exp_id)" - " WHERE" - " detRun.state = 'run'" - ); - - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidExp"); + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + psString query = pxDataGet("dettool_residexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidExp"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (faulted) { @@ -3394,4 +3127,10 @@ } + // list only accepted rows + // XXX the usage of this flag is unclear : -reject means "accepted"? + if (reject) { + psStringAppend(&query, " %s", "AND detResidExp.accept != 0"); + } + // treat limit == 0 as "no limit" if (limit) { @@ -3436,4 +3175,10 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-code", "fault", "=="); + psString query = pxDataGet("dettool_revertresidexp.sql"); if (!query) { @@ -3442,9 +3187,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detResidExp"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detResidExp"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -3464,4 +3210,50 @@ +static bool updateresidexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + + // build a query to search by det_id, iteration, exp_id + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); + + // find the values we're going to set + // copy everything but det_id, iteration, & exp_id from the args and + // remove the '-' prefix + psMetadata *set = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, set, "-recip", "recipe", "=="); + PXOPT_COPY_F64(config->args, set, "-bg", "bg", "=="); + PXOPT_COPY_F64(config->args, set, "-bg_stdev", "bg_stdev", "=="); + PXOPT_COPY_F64(config->args, set, "-bg_mean_stdev", "bg_mean_stdev", "=="); + PXOPT_COPY_STR(config->args, set, "-path_base", "path_base", "=="); + + // this can't be PXOPT_ macro-ized as reject is !'d + if (!psMetadataAddBool(set, PS_LIST_TAIL, "accept", 0, "==", !reject)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item accept"); + psFree(set); + psFree(where); + return false; + } + + long changed = psDBUpdateRows(config->dbh, "detResidExp", where, set); + psFree(set); + psFree(where); + + if (changed < 0) { + psError(PS_ERR_UNKNOWN, false, "no rows were updated"); + return false; + } + + return true; +} + +/* The SQL returns a list of detrend runs (with detrend id, iteration and detrend type) which + * have completed all residexps. + */ + static bool todetrunsummaryMode(pxConfig *config) { @@ -3470,8 +3262,4 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - /* which returns a list of detrend runs (with detrend id, iteration and - * detrend type) which have completed all residexps. - */ psString query = pxDataGet("dettool_todetrunsummary.sql"); @@ -3480,13 +3268,4 @@ return false; } - - // XXX does it make sense to accept any search params? -#if 0 - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } -#endif // treat limit == 0 as "no limit" @@ -3527,56 +3306,58 @@ } -static bool updateresidexpMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); - +// this function is used to generate the detrunSummary entries used below +// XXX why is this a separate function? roll it back into adddetrunsummary? +static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + bool status = false; + // from row + psS64 det_id = psMetadataLookupS64(&status, row, "det_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); + return false; + } + psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); + return false; + } + + // optional + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); + + return detRunSummaryRowAlloc( + det_id, + iteration, + bg, + bg_stdev, + bg_mean_stdev, + accept, + code + ); +} + +static bool adddetrunsummaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // build a query to search by det_id, iteration, exp_id psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_id", "exp_id", "=="); - - // find the values we're going to set - // copy everything but det_id, iteration, & exp_id from the args and - // remove the '-' prefix - psMetadata *set = psMetadataAlloc(); - // XXX if PXOPT_COPY_* errors here the where md will leak - PXOPT_COPY_STR(config->args, set, "-recip", "recipe", "=="); - PXOPT_COPY_F64(config->args, set, "-bg", "bg", "=="); - PXOPT_COPY_F64(config->args, set, "-bg_stdev", "bg_stdev", "=="); - PXOPT_COPY_F64(config->args, set, "-bg_mean_stdev", "bg_mean_stdev", "=="); - PXOPT_COPY_STR(config->args, set, "-path_base", "path_base", "=="); - - // this can't be PXOPT_ macro-ized as reject is !'d - if (!psMetadataAddBool(set, PS_LIST_TAIL, "accept", 0, "==", !reject)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item accept"); - psFree(set); - psFree(where); - return false; - } - - long changed = psDBUpdateRows(config->dbh, "detResidExp", where, set); - psFree(set); - psFree(where); - - if (changed < 0) { - psError(PS_ERR_UNKNOWN, false, "no rows were updated"); - return false; - } - - return true; -} - -static bool adddetrunsummaryMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); - - // optional + + PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); // required PXOPT_LOOKUP_BOOL(again, config->args, "-again", false); + + // The values supplied as arguments on the command (eg, -bg) are parsed + // by mdToDetRunSummary below. + // XXX why is there ever more than one? psString query = pxDataGet("dettool_find_completed_runs.sql"); @@ -3586,18 +3367,10 @@ } - // build a query to search by det_id, iteration, exp_id - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); - PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); - - { - // there's not + if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psFree(where); - if (whereClause) { - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - } - } + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -3626,4 +3399,5 @@ } + // XXX why is there ever more than one result here? for (long i = 0; i < psArrayLength(output); i++) { psMetadata *row = output->data[i]; @@ -3654,4 +3428,5 @@ // XXX this logic does not deal with the case of -code being set + // XXX it should be an error for -again and -code to both be set if (again) { if (!startNewIteration(config, (psS64)atoll(det_id))) { @@ -3680,43 +3455,11 @@ } -static detRunSummaryRow *mdToDetRunSummary(pxConfig *config, psMetadata *row) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - bool status = false; - // from row - psS64 det_id = psMetadataLookupS64(&status, row, "det_id"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for det_id"); - return false; - } - psS32 iteration = psMetadataLookupS32(&status, row, "iteration"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for iteration"); - return false; - } - - // optional - PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); - PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); - PXOPT_LOOKUP_F64(bg_mean_stdev, config->args, "-bg_mean_stdev", false, false); - // default values - PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); - PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - - return detRunSummaryRowAlloc( - det_id, - iteration, - bg, - bg_stdev, - bg_mean_stdev, - accept, - code - ); -} - static bool detrunsummaryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -3724,21 +3467,17 @@ PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - psString query = psStringCopy( - "SELECT DISTINCT\n" - " detRunSummary.*,\n" - " detRun.det_type,\n" - " detRun.mode\n" - " FROM detRun\n" - " JOIN detRunSummary\n" - " USING(det_id, iteration)\n" - " WHERE\n" - " detRun.state = 'run'\n" - ); - - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psString query = pxDataGet("dettool_detrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "NULL"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (faulted) { @@ -3792,4 +3531,9 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-code", "fault", "=="); + psString query = pxDataGet("dettool_revertdetrunsummary.sql"); if (!query) { @@ -3798,9 +3542,10 @@ } - if (config->where) { - psString whereClause = psDBGenerateWhereConditionSQL(config->where, "detRunSummary"); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } + psFree(where); if (!p_psDBRunQuery(config->dbh, query)) { @@ -3823,8 +3568,6 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); - + + PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); // required PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); @@ -3854,11 +3597,8 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // det_id is required - // XXX this isn't strictly nessicary but not having the det_id complicates - // incrementing the iteration number - PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); - + PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); // required PXOPT_LOOKUP_BOOL(again, config->args, "-again", false); PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + // either -rerun or -state must be specified if (!(again || state)) { @@ -3867,5 +3607,5 @@ } if (again && state) { - psError(PS_ERR_UNKNOWN, true, "either -again or -state must be specified"); + psError(PS_ERR_UNKNOWN, true, "-accept and -reject are exclusive"); return false; } @@ -3886,4 +3626,5 @@ } +// used by updatedetrunMode static bool startNewIteration(pxConfig *config, psS64 det_id) { @@ -3896,7 +3637,5 @@ } - // XXX this query was not restricted by det_id, resulting - // in an inconsistent UPDATE below. I added this AND clause - // though there may be a cleaner method (EAM 2006.10.08) + // restrict the update only to the specified det_id psStringAppend(&query, " WHERE det_id = %" PRId64 , det_id); @@ -4151,9 +3890,6 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); - PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); - - // optional + PXOPT_LOOKUP_STR(det_type, config->args, "-det_type", true, false); // required + PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", true, false); // required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); @@ -4178,6 +3914,4 @@ PXOPT_LOOKUP_STR(parent, config->args, "-parent", false, false); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - - // default PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -4191,5 +3925,5 @@ 0, // the iteration is fixed at 0 det_type, - "register", // mode + "register", // mode is required to be "register" "register", // state filelevel, @@ -4262,14 +3996,12 @@ } +// NOTE : this function is also used by addcorrectimfileMode bool register_detrend_imfileMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - - // optional + + PXOPT_LOOKUP_STR(det_id, config->args, "-det_id", true, false); // required + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); // required PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); @@ -4389,2 +4121,95 @@ return true; } + + +#if 0 +// XXX this function was left in commented as this method may be useful in the +// future +static psArray *validDetInputClassIds(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *rawImfiles = searchInputImfiles(config, det_id); + if (!rawImfiles) { + return NULL; + } + + psArray *valid_class_ids = NULL; + { + // All this jumping through hoops is so that we end up with unique + // values. PP thinks that making multiple passes through this array and + // deleting matched elements would end up being more expensive then a + // double sort and stagger scheme. JH thinks it would be cheaper to + // just do a unqiue sort and delete. So this is really just a cheap + // hack to avoid implimenting a unique sort function but at least it's + // stable. + psHash *hash = psHashAlloc(psArrayLength(rawImfiles)); + for (long i = 0; i < psArrayLength(rawImfiles); i++) { + psHashAdd(hash, + ((rawImfileRow *)rawImfiles->data[i])->class_id, + ((rawImfileRow *)rawImfiles->data[i])->class_id + ); + } + valid_class_ids = psHashToArray(hash); + psFree(hash); + } + psFree(rawImfiles); + + return valid_class_ids; +} + +static psArray *searchInputImfiles(pxConfig *config, const char *det_id) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + // det_id is input as a string because the fact that it is an integer + // is just a database impliementation detail. + PS_ASSERT_PTR_NON_NULL(det_id, NULL); + + psArray *inputExps = NULL; + { + psMetadata *where = psMetadataAlloc(); + if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==", + (psS32)atoi(det_id))) { + psError(PS_ERR_UNKNOWN, false, "failed to add item det_id"); + psFree(where); + return NULL; + } + inputExps = detInputExpSelectRowObjects(config->dbh, where, 0); + psFree(where); + } + if (!inputExps) { + psError(PS_ERR_UNKNOWN, false, "no detInputExp rows found"); + return NULL; + } + + // find rawImfiles associated with detInputExps + psArray *rawImfiles = NULL; + { + psMetadata *where = psMetadataAlloc(); + for (long i = 0; i < psArrayLength(inputExps); i++) { + if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", + PS_META_DUPLICATE_OK, "==", + ((detInputExpRow *)inputExps->data[i])->exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(inputExps); + psFree(where); + return NULL; + } + } + psFree(inputExps); + rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0); + // XXX this really should be sorted for uniqueness + psFree(where); + } + if (!rawImfiles) { + psError(PS_ERR_UNKNOWN, false, "no rawImfile rows found"); + return NULL; + } + + return rawImfiles; +} +#endif +