Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 27144) +++ trunk/ippTools/share/Makefile.am (revision 27149) @@ -256,4 +256,8 @@ regtool_revertprocessedimfile.sql \ regtool_updateprocessedimfile.sql \ + regtool_pendingcompressimfile.sql \ + regtool_finishcompressexp.sql \ + regtool_updatebyquery.sql \ + regtool_updatebyqueryimfile.sql \ stacktool_definebyquery_insert.sql \ stacktool_definebyquery_insert_random_part1.sql \ Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 27144) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 27149) @@ -117,4 +117,5 @@ filelevel VARCHAR(64), workdir VARCHAR(255), + state VARCHAR(64), reduction VARCHAR(64), dvodb VARCHAR(255), @@ -189,4 +190,5 @@ class_id VARCHAR(64), uri VARCHAR(255), + data_state VARCHAR(64), exp_type VARCHAR(64), filelevel VARCHAR(64), Index: trunk/ippTools/share/regtool_finishcompressexp.sql =================================================================== --- trunk/ippTools/share/regtool_finishcompressexp.sql (revision 27149) +++ trunk/ippTools/share/regtool_finishcompressexp.sql (revision 27149) @@ -0,0 +1,89 @@ +select DISTINCT + exp_id, + exp_name, + camera, + telescope, + dateobs, + exp_tag, + exp_type, + filelevel, + workdir, + state, + reduction, + dvodb, + tess_id, + end_stage, + filter, + comment, + obs_mode, + obs_group, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + m1_x, + m1_y, + m1_z, + m1_tip, + m1_tilt, + m2_x, + m2_y, + m2_z, + m2_tip, + m2_tilt, + env_temperature, + env_humidity, + env_wind_speed, + env_wind_dir, + teltemp_m1, + teltemp_m1cell, + teltemp_m2, + teltemp_spider, + teltemp_truss, + teltemp_extra, + pon_time, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + sun_angle, + sun_alt, + moon_angle, + moon_alt, + moon_phase, + hostname, + fault, + epoch, + magicked +FROM + (SELECT + rawExp.*, + count(rawImfile.class_id) AS complete, + count(newImfile.tmp_class_id) AS total + FROM rawExp + JOIN newImfile + ON rawExp.exp_id = newImfile.exp_id + LEFT JOIN rawImfile + ON rawExp.exp_id = rawImfile.exp_id + AND rawImfile.tmp_class_id = newImfile.tmp_class_id + WHERE + ((rawExp.state = 'goto_compressed' + AND rawImfile.data_state = 'compressed')OR + (rawExp.state = 'goto_lossy' + AND rawImfile.data_state = 'lossy')) + GROUP BY + rawExp.exp_id + ) as foo + WHERE (foo.complete = foo.total) + + Index: trunk/ippTools/share/regtool_pendingcompressimfile.sql =================================================================== --- trunk/ippTools/share/regtool_pendingcompressimfile.sql (revision 27149) +++ trunk/ippTools/share/regtool_pendingcompressimfile.sql (revision 27149) @@ -0,0 +1,4 @@ +SELECT * from rawImfile + WHERE (data_state = 'goto_compressed' OR data_state = 'goto_lossy') +-- where hook %s +-- limit hook %s Index: trunk/ippTools/share/regtool_updatebyquery.sql =================================================================== --- trunk/ippTools/share/regtool_updatebyquery.sql (revision 27149) +++ trunk/ippTools/share/regtool_updatebyquery.sql (revision 27149) @@ -0,0 +1,5 @@ +UPDATE rawExp + SET state = '%s' +WHERE 1 +-- where hook %s +-- limit hook %s Index: trunk/ippTools/share/regtool_updatebyqueryimfile.sql =================================================================== --- trunk/ippTools/share/regtool_updatebyqueryimfile.sql (revision 27149) +++ trunk/ippTools/share/regtool_updatebyqueryimfile.sql (revision 27149) @@ -0,0 +1,5 @@ +UPDATE rawImfile JOIN rawExp USING(exp_id) + SET data_state = '%s' +WHERE 1 +-- where hook %s +-- limit hook %s Index: trunk/ippTools/share/regtool_updateprocessedimfile.sql =================================================================== --- trunk/ippTools/share/regtool_updateprocessedimfile.sql (revision 27144) +++ trunk/ippTools/share/regtool_updateprocessedimfile.sql (revision 27149) @@ -1,4 +1,4 @@ UPDATE rawImfile - SET burntool_state = %d + SET %s WHERE rawImfile.exp_id = %lld Index: trunk/ippTools/src/pxregister.c =================================================================== --- trunk/ippTools/src/pxregister.c (revision 27144) +++ trunk/ippTools/src/pxregister.c (revision 27149) @@ -58,2 +58,67 @@ return true; } + +bool pxrawExpSetState(pxConfig *config, psS64 exp_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "full", 5) == 0) + || (strncmp(state, "new", 4) == 0) + || (strncmp(state, "goto_compressed", 14) == 0) + || (strncmp(state, "compressed", 11) == 0) + || (strncmp(state, "goto_lossy", 11) == 0) + || (strncmp(state, "lossy", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid rawExp state: %s", state); + return false; + } + + char *query = "UPDATE rawExp SET state = '%s' WHERE exp_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} +bool pxrawImfileSetState(pxConfig *config, psS64 exp_id, const char *class_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "full", 5) == 0) + || (strncmp(state, "new", 4) == 0) + || (strncmp(state, "goto_compressed", 14) == 0) + || (strncmp(state, "compressed", 11) == 0) + || (strncmp(state, "goto_lossy", 11) == 0) + || (strncmp(state, "lossy", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid rawImfile data_state: %s", state); + return false; + } + + char *query = "UPDATE rawImfile SET data_state = '%s' WHERE class_id = %s AND exp_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, class_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} Index: trunk/ippTools/src/pxregister.h =================================================================== --- trunk/ippTools/src/pxregister.h (revision 27144) +++ trunk/ippTools/src/pxregister.h (revision 27149) @@ -26,4 +26,6 @@ bool pxnewExpSetState(pxConfig *config, psS64 exp_id, const char *state); +bool pxrawExpSetState(pxConfig *config, psS64 exp_id, const char *state); +bool pxrawImfileSetState(pxConfig *config, psS64 exp_id, const char *class_id, const char *state); #endif // PXREGISTER_H Index: trunk/ippTools/src/regtool.c =================================================================== --- trunk/ippTools/src/regtool.c (revision 27144) +++ trunk/ippTools/src/regtool.c (revision 27149) @@ -37,4 +37,5 @@ static bool revertprocessedimfileMode(pxConfig *config); static bool updateprocessedimfileMode(pxConfig *config); +static bool pendingcompressimfileMode(pxConfig *config); // exp static bool pendingexpMode(pxConfig *config); @@ -44,4 +45,8 @@ static bool updateprocessedexpMode(pxConfig *config); static bool cleardupexpMode(pxConfig *config); +static bool finishcompressexpMode(pxConfig *config); + +static bool updatebyqueryMode(pxConfig *config); + static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -77,4 +82,7 @@ MODECASE(REGTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(REGTOOL_MODE_UPDATEBYQUERY, updatebyqueryMode); + MODECASE(REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileMode); + MODECASE(REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpMode); MODECASE(REGTOOL_MODE_CLEARDUPEXP, cleardupexpMode); MODECASE(REGTOOL_MODE_EXPORTRUN, exportrunMode); @@ -189,4 +197,5 @@ PXOPT_LOOKUP_STR(obs_mode, config->args, "-obs_mode", false, false); PXOPT_LOOKUP_STR(obs_group, config->args, "-obs_group", false, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false); PXOPT_LOOKUP_F32(airmass, config->args, "-airmass", false, false); @@ -255,4 +264,5 @@ class_id, uri, + data_state, exp_type, filelevel, @@ -336,5 +346,5 @@ PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "=="); - PXOPT_COPY_S64(config->args, where, "-magicked", "magicked", "=="); + PXOPT_COPY_STR(config->args, where, "-data_state", "data_state", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -491,9 +501,11 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); -/* PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); */ PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); - - if ((fault == INT16_MAX) && !isfinite(burntool_state)) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state must be selected"); + PXOPT_LOOKUP_S32(set_bytes, config->args, "-set_bytes", false, false); + PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + + if ((fault == INT16_MAX) && !isfinite(burntool_state) && !(set_state)) { + psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); return false; } @@ -514,25 +526,122 @@ return false; } - } - - if (isfinite(burntool_state)) { - psString query = pxDataGet("regtool_updateprocessedimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, burntool_state, exp_id, class_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); + psFree (where); + return(true); } psFree (where); + + psString setvalues = NULL; + if (burntool_state != INT16_MAX) { + psStringAppend(&setvalues,"rawImfile.burntool_state = %d",burntool_state); + } + if (set_bytes != INT32_MAX) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.bytes = %d",set_bytes); + } + if (set_md5sum) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.md5sum = '%s'",set_md5sum); + } + if (set_state) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state); + } + + psString query = pxDataGet("regtool_updateprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // printf(query,setvalues,exp_id,class_id); + if (!p_psDBRunQueryF(config->dbh, query, setvalues, exp_id,class_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(setvalues); + return false; + } + psFree(setvalues); + psFree(query); return true; } +static bool pendingcompressimfileMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (whereClause) { + psStringPrepend(&whereClause, "\n AND "); + } + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_pendingcompressimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + // printf(query,whereClause,limitString); + if (!p_psDBRunQueryF(config->dbh, query, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingCompressImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + static bool pendingexpMode(pxConfig *config) @@ -626,4 +735,5 @@ PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); @@ -787,4 +897,5 @@ filelevel, workdir, + state, reduction, dvodb, @@ -930,6 +1041,7 @@ PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); @@ -964,5 +1076,5 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); - + psString where2 = NULL; if (!pxspaceAddWhere(config, &where2, "rawExp")) { @@ -974,12 +1086,12 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - + if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); } psFree(where); - + if (where2) { psStringAppend(&query, " %s ", where2); @@ -988,55 +1100,55 @@ if (faulted) { - // list only faulted rows - psStringAppend(&query, " %s", "AND rawExp.fault != 0"); + // list only faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault != 0"); } else { - // don't list faulted rows - psStringAppend(&query, " %s", "AND rawExp.fault = 0"); - } - + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault = 0"); + } + // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; } psFree(query); - + psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; } if (!psArrayLength(output)) { - psTrace("regtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); - + return true; } @@ -1044,42 +1156,42 @@ static bool revertprocessedexpMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id_begin", "exp_id", ">="); - PXOPT_COPY_S64(config->args, where, "-exp_id_end", "exp_id", "<="); - - psString query = pxDataGet("regtool_revertprocessedexp.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - psFree(where); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } else { - psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; - } + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id_begin", "exp_id", ">="); + PXOPT_COPY_S64(config->args, where, "-exp_id_end", "exp_id", "<="); + + psString query = pxDataGet("regtool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - - int numUpdated = psDBAffectedRows(config->dbh); - - psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated); - - return true; + return false; + } + psFree(query); + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated); + + return true; } @@ -1087,21 +1199,205 @@ static bool updateprocessedexpMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - - if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - psFree(where); - return false; - } + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree(where); - - return true; -} - + return false; + } + psFree(where); + + return true; +} + + +static bool finishcompressexpMode(pxConfig *config) { + + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_finishcompressexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " %s ", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s ", limitString); + psFree(limitString); + } + // printf(query); + // return(false); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + rawExpRow *row = rawExpObjectFromMetadata(md); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (strcmp(row->state,"goto_compressed") == 0) { + if (!pxrawExpSetState(config,row->exp_id, "compressed")) { + psError(PS_ERR_UNKNOWN, false, "failed to set rawExp.state for exp_id: %" PRId64, row->exp_id); + psFree(row); + psFree(output); + return(false); + } + } + else if (strcmp(row->state,"goto_lossy") == 0) { + if (!pxrawExpSetState(config,row->exp_id, "lossy")) { + psError(PS_ERR_UNKNOWN, false, "failed to set rawExp.state for exp_id: %" PRId64, row->exp_id); + psFree(row); + psFree(output); + return(false); + } + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(row); + } + + psFree(output); + return(true); +} + +static bool updatebyqueryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", true, false); + + // XX test this out; need to make this consistent with the list in regtoolConfig.c + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause,"\n AND "); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_updatebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + // printf(query,set_state,whereClause,limitString); + if (!p_psDBRunQueryF(config->dbh, query, set_state, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + + psFree(query); + // Now up date the imfiles. + query = pxDataGet("regtool_updatebyqueryimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, set_state,whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + + return true; +} + + static bool cleardupexpMode(pxConfig *config) Index: trunk/ippTools/src/regtool.h =================================================================== --- trunk/ippTools/src/regtool.h (revision 27144) +++ trunk/ippTools/src/regtool.h (revision 27149) @@ -36,4 +36,7 @@ REGTOOL_MODE_UPDATEPROCESSEDEXP, REGTOOL_MODE_CLEARDUPEXP, + REGTOOL_MODE_UPDATEBYQUERY, + REGTOOL_MODE_PENDINGCOMPRESSIMFILE, + REGTOOL_MODE_FINISHCOMPRESSEXP, REGTOOL_MODE_EXPORTRUN, REGTOOL_MODE_IMPORTRUN Index: trunk/ippTools/src/regtoolConfig.c =================================================================== --- trunk/ippTools/src/regtoolConfig.c (revision 27144) +++ trunk/ippTools/src/regtoolConfig.c (revision 27149) @@ -72,6 +72,6 @@ ADD_OPT(Str, addprocessedimfileArgs, "-comment", "define comment ", NULL); ADD_OPT(Str, addprocessedimfileArgs, "-obs_mode", "define observing mode (data usage goal)", NULL); - ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); - + ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-data_state", "define the data_state", "full"); // Note: RA & DEC are supplied here in radians, but the query options use degrees. This // can be justified by the fact that this option (addprocessedimfile) is a software @@ -139,4 +139,5 @@ ADD_OPT(Str, processedimfileArgs, "-obs_mode", "search by obs_mod", NULL); ADD_OPT(Str, processedimfileArgs, "-exp_type", "search by exposure type", NULL); + ADD_OPT(Str, processedimfileArgs, "-data_state", "search by data_state", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL); @@ -162,7 +163,11 @@ ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); - ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", 0); + ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", INT16_MAX); ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); - + ADD_OPT(S32, updateprocessedimfileArgs, "-set_bytes", "set bytes", INT32_MAX); + ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum", "set md5sum", NULL); + ADD_OPT(Str, updateprocessedimfileArgs, "-set_state", "set data state", NULL); + + // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); @@ -185,4 +190,5 @@ ADD_OPT(Str, addprocessedexpArgs, "-exp_type", "define exposure type", NULL); ADD_OPT(Str, addprocessedexpArgs, "-workdir", "define the \"default\" workdir for this exposure", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-state", "define the state for this exposure", NULL); ADD_OPT(Str, addprocessedexpArgs, "-dvodb", "define the dvodb for the next processing step", NULL); ADD_OPT(Str, addprocessedexpArgs, "-tess_id", "define the tess_id for the next processing step", NULL); @@ -253,4 +259,5 @@ psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-state", 0, "search by exposure state", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); @@ -302,5 +309,5 @@ psMetadataAddS64(updatedprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", INT16_MAX); - + psMetadataAddStr(updatedprocessedexpArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -313,8 +320,71 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - // -cleardupexp psMetadata *cleardupexpArgs = psMetadataAlloc(); + // -updatebyquery + psMetadata *updatebyqueryArgs = psMetadataAlloc(); + + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "set the state", NULL); + + psMetadataAddS64(updatebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(updatebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(updatebyqueryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by obs_mode", NULL); + + psMetadataAddU64(updatebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(updatebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + + // -pendingcompressimfile + psMetadata *pendingcompressimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcompressimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(pendingcompressimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcompressimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + + // -finishcompressexp + psMetadata *finishcompressexpArgs = psMetadataAlloc(); + psMetadataAddS64(finishcompressexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddBool(finishcompressexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(finishcompressexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -331,4 +401,7 @@ PXOPT_ADD_MODE("-updateprocessedexp", "", REGTOOL_MODE_UPDATEPROCESSEDEXP, updatedprocessedexpArgs); PXOPT_ADD_MODE("-cleardupexp", "", REGTOOL_MODE_CLEARDUPEXP, cleardupexpArgs); + PXOPT_ADD_MODE("-updatebyquery", "", REGTOOL_MODE_UPDATEBYQUERY, updatebyqueryArgs); + PXOPT_ADD_MODE("-pendingcompressimfile", "", REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileArgs); + PXOPT_ADD_MODE("-finishcompressexp", "", REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", REGTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", REGTOOL_MODE_IMPORTRUN, importrunArgs);