Index: branches/simtest_nebulous_branches/ippTools/src/Makefile.am =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/Makefile.am (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/Makefile.am (revision 27840) @@ -1,3 +1,4 @@ bin_PROGRAMS = \ + addtool \ caltool \ camtool \ @@ -8,4 +9,5 @@ disttool \ flatcorr \ + dqstatstool \ magictool \ magicdstool \ @@ -23,9 +25,6 @@ pubtool - -bin_SCRIPTS = \ - fakemagic - pkginclude_HEADERS = \ + pxadd.h \ pxadmin.h \ pxcam.h \ @@ -34,5 +33,8 @@ pxdata.h \ pxfake.h \ + pxdqstats.h + pxmagic.h \ pxregister.h \ + pxspace.h \ pxtag.h \ pxtools.h \ @@ -42,4 +44,5 @@ noinst_HEADERS = \ + addtool.h \ caltool.h \ camtool.h \ @@ -51,4 +54,5 @@ flatcorr.h \ faketool.h \ + dqstatstool.h \ magictool.h \ magicdstool.h \ @@ -68,4 +72,5 @@ libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) libpxtools_la_SOURCES = \ + pxadd.c \ pxcam.c \ pxchip.c \ @@ -75,5 +80,8 @@ pxfake.c \ pxfault.c \ + pxdqstats.c \ + pxmagic.c \ pxregister.c \ + pxspace.c \ pxtag.c \ pxtools.c \ @@ -97,4 +105,10 @@ pstamptoolConfig.c +addtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +addtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +addtool_SOURCES = \ + addtool.c \ + addtoolConfig.c + caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) caltool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la @@ -132,4 +146,10 @@ faketool.c \ faketoolConfig.c + +dqstatstool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +dqstatstool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +dqstatstool_SOURCES = \ + dqstatstool.c \ + dqstatstoolConfig.c magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: branches/simtest_nebulous_branches/ippTools/src/addtool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/addtool.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/addtool.c (revision 27840) @@ -0,0 +1,734 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = addtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(ADDTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(ADDTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(ADDTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(ADDTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(ADDTOOL_MODE_BLOCK, blockMode); + MODECASE(ADDTOOL_MODE_MASKED, maskedMode); + MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","camRun.data_group", "=="); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the cam_id of all the exposures that we want to queue up. + psString bare_query = pxDataGet("addtool_find_cam_id.sql"); + if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // prevent queueing an addRun if a given exposure has already been added to + // the given dvo database + psString dvodb_string = NULL; + if (dvodb) { + // user supplied dvodb + psStringAppend(&dvodb_string, "(previous_dvodb = '%s')", dvodb); + } else { + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(camRun.dvodb IS NOT NULL AND previous_dvodb = camRun.dvodb)"); + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + psFree(where); + + if (destreaked) { + psStringAppend(&query, " AND (camRun.magicked > 0)"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + camRunRow *row = camRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb && !row->dvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, cam_id %" PRId64, row->label, row->cam_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, cam_id %" PRId64, row->label, row->cam_id); + psFree(output); + return false; + } + + psFree(row); + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + // loop over our list of camRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + camRunRow *row = camRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + row->cam_id, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : row->dvodb, + note ? note : NULL, + image_only + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); // most search arguments based on camera + PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-dvodb", "addRun.dvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and runs the update query + bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", + "addProcessedExp", false); + + psFree(query); + psFree(where); + + return result; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + addRunRow *pendingRow = addRunObjectFromMetadata(output->data[0]); + psFree(output); + addProcessedExpRow *row = addProcessedExpRowAlloc( + pendingRow->add_id, + dtime_addstar, + path_base, + fault + ); + + if (!addProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'new' set addRun.state = 'full' + if (!pxaddRunSetState(config, row->add_id, "full")) { + psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); + psFree(row); + psFree(pendingRow); + return false; + } + psFree(row); + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); + return false; + } + + psString query = pxDataGet("addtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + // we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault = 0"); + } + psFree(where); + + // order by add_id so that the postage stamp parser can easliy find the 'latest' astrometry + psStringAppend(&query, " ORDER BY add_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("addtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + + { + psString query = pxDataGet("addtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!addMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM addMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM addMask WHERE label = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/addtool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/addtool.h (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/addtool.h (revision 27840) @@ -0,0 +1,46 @@ +/* + * addtool.h + * + * Copyright (C) 2006 Joshua Hoblitt, Christopher Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef ADDTOOL_H +#define ADDTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + ADDTOOL_MODE_NONE = 0x0, + ADDTOOL_MODE_DEFINEBYQUERY, + ADDTOOL_MODE_UPDATERUN, + ADDTOOL_MODE_PENDINGEXP, + ADDTOOL_MODE_ADDPROCESSEDEXP, + ADDTOOL_MODE_PROCESSEDEXP, + ADDTOOL_MODE_REVERTPROCESSEDEXP, + ADDTOOL_MODE_UPDATEPROCESSEDEXP, + ADDTOOL_MODE_BLOCK, + ADDTOOL_MODE_MASKED, + ADDTOOL_MODE_UNBLOCK, + ADDTOOL_MODE_PENDINGCLEANUPRUN, + ADDTOOL_MODE_PENDINGCLEANUPEXP, + ADDTOOL_MODE_DONECLEANUP, + ADDTOOL_MODE_EXPORTRUN, + ADDTOOL_MODE_IMPORTRUN +} addtoolMode; + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // ADDTOOL_H Index: branches/simtest_nebulous_branches/ippTools/src/addtoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/addtoolConfig.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/addtoolConfig.c (revision 27840) @@ -0,0 +1,178 @@ +/* + * addtoolConfig.c + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by camRun data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-destreaked", 0, "only queue destreaked runs", false); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-image_only", 0, "addstar image metadata but not detections", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(updaterunArgs); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by addRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by addRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by addRun reduction class", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dvodb", 0, "search by dvodb", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dvodb", 0, "set dvodb", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by addRun reduction class", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(processedexpArgs); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); + + // -revertprocessedexp + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + pxcamSetSearchArgs(revertprocessedexpArgs); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updateprocessedexp + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change add run properties", ADDTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exps", ADDTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "add a processed exp", ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "show processed exps", ADDTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exp", ADDTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "set a label block", ADDTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/simtest_nebulous_branches/ippTools/src/caltoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/caltoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/caltoolConfig.c (revision 27840) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; Index: branches/simtest_nebulous_branches/ippTools/src/camtool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/camtool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/camtool.c (revision 27840) @@ -109,11 +109,11 @@ psMetadata *where = psMetadataAlloc(); pxcamGetSearchArgs (config, where); - pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); // define using chipRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -121,4 +121,6 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); @@ -126,9 +128,10 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // find the exp_id of all the exposures that we want to queue up. psString query = pxDataGet("camtool_find_chip_id.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -181,8 +184,8 @@ bool status; char *end_stage = psMetadataLookupStr(&status, md, "end_stage"); - if (end_stage && strcasecmp(end_stage, "warp")) continue; + if (end_stage && strcasecmp(end_stage, "warp")) continue; char *raw_tess_id = psMetadataLookupStr(&status, md, "tess_id"); - if (raw_tess_id || tess_id) continue; + if (raw_tess_id || tess_id) continue; char *label = psMetadataLookupStr(&status, md, "label"); @@ -190,5 +193,5 @@ if (!status) { - psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); + psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); psFree(output); return false; @@ -196,11 +199,11 @@ } - // loop over our list of camRun rows + // loop over our list of chipRun rows for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; - camRunRow *row = camRunObjectFromMetadata(md); + chipRunRow *row = chipRunObjectFromMetadata(md); if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); psFree(output); return false; @@ -212,9 +215,13 @@ workdir ? workdir : row->workdir, label ? label : row->label, + data_group ? data_group: row->data_group, + dist_group ? dist_group: row->dist_group, reduction ? reduction : row->reduction, expgroup ? expgroup : row->expgroup, dvodb ? dvodb : row->dvodb, tess_id ? tess_id : row->tess_id, - end_stage ? end_stage : row->end_stage + end_stage ? end_stage : row->end_stage, + row->magicked, + note )) { if (!psDBRollback(config->dbh)) { @@ -248,42 +255,25 @@ PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group","camRun.data_group", "=="); PXOPT_COPY_STR(config->args, where, "-state", "camRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set camRun.state to state - if (!pxcamRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set camRun.label to label - if (!pxcamRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "camRun", "cam_id", "camProcessedExp", true); + if (!result) { + psError(psErrorCodeLast(), false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -304,5 +294,5 @@ psString query = pxDataGet("camtool_find_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -369,5 +359,5 @@ psString query = pxDataGet("camtool_find_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -491,6 +481,7 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); - - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + +// Get this from the chipRun +// PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); // generate restrictions @@ -500,5 +491,5 @@ psString query = pxDataGet("camtool_find_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -535,4 +526,7 @@ return false; } + + bool status; + psS64 chip_magicked = psMetadataLookupS64(&status, output->data[0], "chip_magicked"); camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]); @@ -615,16 +609,5 @@ } - // since there is only one exp per 'new' set camRun.state = 'full' - if (!pxcamRunSetState(config, row->cam_id, "full", magicked)) { - psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); - psFree(row); - psFree(pendingRow); - return false; - } - - // NULL for end_stage means go as far as possible - // EAM : skip here if fault != 0 - // Also, we can run fake even if tess_id is not defined - if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) { + if (fault) { psFree(row); psFree(pendingRow); @@ -635,6 +618,26 @@ return true; } + // else continue on... + + // since there is only one exp per 'new' set camRun.state = 'full' + // propagate magicked state from chipRun + if (!pxcamRunSetState(config, row->cam_id, "full", chip_magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id); + psFree(row); + psFree(pendingRow); + return false; + } psFree(row); - // else continue on... + + // EAM: NULL for end_stage means go as far as possible + // Also, we can run fake even if tess_id is not defined + if (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam")) { + psFree(pendingRow); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } if (!pxfakeQueueByCamID(config, @@ -642,9 +645,12 @@ pendingRow->workdir, pendingRow->label, + pendingRow->data_group, + pendingRow->dist_group, pendingRow->reduction, pendingRow->expgroup, pendingRow->dvodb, pendingRow->tess_id, - pendingRow->end_stage + pendingRow->end_stage, + NULL // note does not propagate )) { // rollback @@ -656,4 +662,5 @@ return false; } + psFree(pendingRow); @@ -674,4 +681,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); // generate restrictions @@ -680,10 +688,20 @@ PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-data_group", "camRun.data_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - if (!psListLength(where->list) && + psString where2 = NULL; + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + if (!pxmagicAddWhere(config, &where2, "chipRun")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + if (!psListLength(where->list) && !where2 && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); return false; } @@ -691,5 +709,5 @@ psString query = pxDataGet("camtool_find_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -699,5 +717,5 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } + } // we either add AND (condition) or WHERE (condition): @@ -705,5 +723,5 @@ // list only faulted rows psStringAppend(&query, " %s", " AND camProcessedExp.fault != 0"); - } + } if (where->list && !faulted) { // don't list faulted rows @@ -713,5 +731,5 @@ // list only faulted rows psStringAppend(&query, " %s", " WHERE camProcessedExp.fault != 0"); - } + } if (!where->list && !faulted) { // don't list faulted rows @@ -719,7 +737,15 @@ } psFree(where); - - // order by cam_id so that the postage stamp parser can easliy find the 'latest' astrometry - psStringAppend(&query, " ORDER BY cam_id"); + if (where2) { + psStringAppend(&query, " %s", where2); + psFree(where2); + } + + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, " ORDER BY exp_id, cam_id DESC"); + } // treat limit == 0 as "no limit" @@ -774,5 +800,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -787,5 +813,5 @@ psString query = pxDataGet("camtool_reset_faulted_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -819,5 +845,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -959,5 +985,5 @@ psString query = pxDataGet("camtool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1024,5 +1050,5 @@ psString query = pxDataGet("camtool_pendingcleanupexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1085,5 +1111,5 @@ psString query = pxDataGet("camtool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1155,4 +1181,9 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); @@ -1166,5 +1197,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1235,6 +1266,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "camRun"); Index: branches/simtest_nebulous_branches/ippTools/src/camtoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/camtoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/camtoolConfig.c (revision 27840) @@ -42,5 +42,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -62,5 +62,7 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); // -updaterun @@ -71,11 +73,14 @@ pxcamSetSearchArgs(updaterunArgs); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by camRun data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); @@ -174,5 +179,10 @@ psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by camRun data_group (LIKE comparison)", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by camRun reduction class", NULL); + pxspaceAddArguments(processedexpArgs); + pxmagicAddArguments(processedexpArgs); + + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order the results for the postage stamp server", false); psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/simtest_nebulous_branches/ippTools/src/chiptool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/chiptool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/chiptool.c (revision 27840) @@ -36,4 +36,5 @@ static bool definebyqueryMode(pxConfig *config); +static bool definecopyMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool pendingimfileMode(pxConfig *config); @@ -54,6 +55,10 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); +static bool setimfiletoupdateMode(pxConfig *config); +static bool listrunMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -75,4 +80,5 @@ switch (config->mode) { MODECASE(CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(CHIPTOOL_MODE_DEFINECOPY, definecopyMode); MODECASE(CHIPTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileMode); @@ -93,6 +99,10 @@ MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(CHIPTOOL_MODE_RUNSTATE, runstateMode); + MODECASE(CHIPTOOL_MODE_SETIMFILETOUPDATE, setimfiletoupdateMode); + MODECASE(CHIPTOOL_MODE_LISTRUN, listrunMode); default: psAbort("invalid option (this should not happen)"); @@ -116,4 +126,116 @@ } +// Queue exposures for chip processing +static bool queue_exposures(pxConfig *config, // Configuration + const psArray *exps, // Exposures information + const char *workdir, // Working directory, or NULL to inherit + const char *label, // Label, or NULL to inherit + const char *data_group, // Data group, or NULL to inherit + const char *dist_group, // Distribution group, or NULL to inherit + const char *reduction, // Reduction class, or NULL to inherit + const char *expgroup, // Exposure group + const char *dvodb, // DVO database, or NULL to inherit + const char *tess_id, // Tessellation identifier, or NULL to inherit + const char *end_stage, // End stage, or NULL to inherit + const char *note // Note + ) +{ + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // if end_stage is warp (or NULL), check for valid tess_id + for (long i = 0; i < psArrayLength(exps); i++) { + psMetadata *md = exps->data[i]; + + bool status; + char *end_stage = psMetadataLookupStr(&status, md, "end_stage"); + if (end_stage && strcasecmp(end_stage, "warp")) continue; + + char *raw_tess_id = psMetadataLookupStr(&status, md, "tess_id"); + if (raw_tess_id || tess_id) continue; + + char *label = psMetadataLookupStr(&status, md, "label"); + psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + + if (!status) { + psError(PS_ERR_UNKNOWN, false, + "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, + label, exp_id); + return false; + } + } + + +# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME) \ + PTYPE VALUE; \ + { \ + bool status; \ + VALUE = psMetadataLookup##CTYPE(&status, md, NAME); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \ + return false; \ + } \ + } + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(exps); i++) { + psMetadata *md = exps->data[i]; + + rawExpRow *row = rawExpObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + return false; + } + + GET_VALUE (psS64, S64, exp_id, "exp_id"); + GET_VALUE (psString, Str, raw_workdir, "workdir"); + GET_VALUE (psString, Str, raw_label, "label"); + GET_VALUE (psString, Str, raw_reduction, "reduction"); + // GET_VALUE (psString, Str, raw_expgroup, "expgroup"); + GET_VALUE (psString, Str, raw_dvodb, "dvodb"); + GET_VALUE (psString, Str, raw_tess_id, "tess_id"); + GET_VALUE (psString, Str, raw_end_stage, "end_stage"); + + if (!row->exp_id) { + psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id"); + return false; + } + + // queue the exp + if (!pxchipQueueByExpTag(config, + exp_id, + workdir ? workdir : raw_workdir, + label ? label : raw_label, + data_group ? data_group : (label ? label : raw_label), + dist_group, + reduction ? reduction : raw_reduction, + // expgroup ? expgroup : raw_expgroup, + // XXX how does expgroup get defined? + expgroup, + dvodb ? dvodb : raw_dvodb, + tess_id ? tess_id : raw_tess_id, + end_stage ? end_stage : raw_end_stage, + note + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} static bool definebyqueryMode(pxConfig *config) @@ -125,5 +247,5 @@ psMetadata *where = psMetadataAlloc(); pxchipGetSearchArgs (config, where); // rawExp only - pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); // define using newExp label // psListLength(where->list) is at least 1 because exp_type defaults to "object" @@ -131,5 +253,5 @@ if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -142,4 +264,7 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -150,5 +275,5 @@ psString query = pxDataGet("chiptool_find_rawexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -190,98 +315,98 @@ } - // start a transaction so we don't end up with an exp without any associted - // imfiles - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - return false; - } - - // if end_stage is warp (or NULL), check for valid tess_id - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - bool status; - char *end_stage = psMetadataLookupStr(&status, md, "end_stage"); - if (end_stage && strcasecmp(end_stage, "warp")) continue; - - char *raw_tess_id = psMetadataLookupStr(&status, md, "tess_id"); - if (raw_tess_id || tess_id) continue; - - char *label = psMetadataLookupStr(&status, md, "label"); - psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); - - if (!status) { - psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); + if (!queue_exposures(config, output, workdir, label, data_group, dist_group, reduction, expgroup, + dvodb, tess_id, end_stage, note)) { + psError(psErrorCodeLast(), false, "Unable to queue exposures for chip."); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool definecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); + + // psListLength(where->list) is at least 1 because exp_type defaults to "object" + // so we require a list longer than 1 entry + if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + // default + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("chiptool_definecopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); return false; } - } - - -# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME) \ - PTYPE VALUE; \ - { bool status; \ - VALUE = psMetadataLookup##CTYPE(&status, md, NAME); \ - if (!status) { \ - psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \ - psFree(output); \ - return false; \ - } } - - // loop over our list of exp_ids - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - rawExpRow *row = rawExpObjectFromMetadata(md); - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); - psFree(output); - return false; - } - - GET_VALUE (psS64, S64, exp_id, "exp_id"); - GET_VALUE (psString, Str, raw_workdir, "workdir"); - GET_VALUE (psString, Str, raw_label, "label"); - GET_VALUE (psString, Str, raw_reduction, "reduction"); - // GET_VALUE (psString, Str, raw_expgroup, "expgroup"); - GET_VALUE (psString, Str, raw_dvodb, "dvodb"); - GET_VALUE (psString, Str, raw_tess_id, "tess_id"); - GET_VALUE (psString, Str, raw_end_stage, "end_stage"); - - if (!row->exp_id) { - psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id"); - psFree(output); - return false; - } - - // queue the exp - if (!pxchipQueueByExpTag(config, - exp_id, - workdir ? workdir : raw_workdir, - label ? label : raw_label, - reduction ? reduction : raw_reduction, - // expgroup ? expgroup : raw_expgroup, - // XXX how does expgroup get defined? - expgroup, - dvodb ? dvodb : raw_dvodb, - tess_id ? tess_id : raw_tess_id, - end_stage ? end_stage : raw_end_stage - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue exp_id: %" PRId64, exp_id); - psFree(output); - return false; - } + psFree(output); + return true; + } + + if (!queue_exposures(config, output, workdir, label, data_group, dist_group, reduction, expgroup, + dvodb, tess_id, end_stage, note)) { + psError(psErrorCodeLast(), false, "Unable to queue exposures for chip."); + psFree(output); + return false; } psFree(output); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } return true; @@ -299,41 +424,24 @@ PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "chipRun.dist_group", "=="); + + if (!psListLength(where->list)) { psFree(where); - where = NULL; - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set chipRun.state to state - if (!pxchipRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set chipRun.label to label - if (!pxchipRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) -- join hook %s\n"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "chipRun", "chip_id", "chipProcessedImfile", true); + if (!result) { + psError(psErrorCodeLast(), false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -353,5 +461,5 @@ psString query = pxDataGet("chiptool_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -593,4 +701,8 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(allfiles, config->args, "-allfiles", false); + if (allfiles) { + faulted = false; + } psMetadata *where = psMetadataAlloc(); @@ -601,10 +713,12 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); - PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { - psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "=="); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipProcessedImfile"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); return false; } @@ -612,12 +726,18 @@ psString query = pxDataGet("chiptool_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || (faulted || where2)) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); + return false; } psFree(where); @@ -626,5 +746,5 @@ // list only faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0"); - } else { + } else if (!allfiles) { // don't list faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); @@ -683,5 +803,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -689,5 +809,11 @@ psString query = pxDataGet("chiptool_revertprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + psString query_update = pxDataGet("chiptool_revertupdatedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -697,4 +823,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_update, " AND %s", whereClause); psFree(whereClause); } @@ -708,4 +835,10 @@ } psFree(query); + if (!p_psDBRunQuery(config->dbh, query_update)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_update); + return false; + } + psFree(query_update); return true; @@ -719,11 +852,26 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - - if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) { + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); + if (psListLength(where->list) == 0) { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + + if (!state) { + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); return false; - } - psFree(where); + } + psFree(where); + } + else { + if (!pxchipProcessedImfileSetStateByQuery(config,where,state)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipProcessedImfile state"); + return(false); + } + } + return true; @@ -828,5 +976,5 @@ psString query = pxDataGet("chiptool_unmasked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -906,5 +1054,5 @@ psString query = pxDataGet("chiptool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -970,5 +1118,5 @@ psString query = pxDataGet("chiptool_pendingcleanupimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1031,5 +1179,5 @@ psString query = pxDataGet("chiptool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1089,9 +1237,12 @@ // make sure that the state string is valid if (!pxIsValidState(state)) { - psError(PXTOOLS_ERR_DATA, false, "%s is not a valid state", state); + psError(PXTOOLS_ERR_CONFIG, false, "%s is not a valid state", state); return false; } psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); @@ -1099,5 +1250,5 @@ psString query = pxDataGet("chiptool_run.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1161,5 +1312,5 @@ psString query = pxDataGet("chiptool_completely_processed_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1233,9 +1384,13 @@ chipRun->workdir, chipRun->label, + chipRun->data_group, + chipRun->dist_group, chipRun->reduction, chipRun->expgroup, chipRun->dvodb, chipRun->tess_id, - chipRun->end_stage + chipRun->end_stage, + chipRun->magicked, + NULL // note does not propagate )) { if (!psDBRollback(config->dbh)) { @@ -1278,13 +1433,24 @@ } - psString set_magic = ""; + psString set_magicked_imfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); if (!strcmp(data_state, "full")) { - // copy the magicked state from the input to the output when transitioning to full state - set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked"; - } - - // note only updates if chipRun.state = run_state - // XXX note that we have removed this constraint for now - if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) { + // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will + // block warp processing until destreaking has been done + // otherwise copy magicked from the rawImfile + // Same thing for chipRun/rawExp + psStringAppend(&set_magicked_imfile, "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0" + " AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)"); + psStringAppend(&set_magicked_run, "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), " + " chipRun.magicked, rawExp.magicked)"); + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + // if magicked is non-zero set it to -1 + // Once one imfile has been cleaned, the chipRun is no longer 'magicked' + psStringAppend(&set_magicked_imfile, "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)," + " chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) { psFree(query); psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1297,4 +1463,6 @@ } psFree(query); + psFree(set_magicked_imfile); + psFree(set_magicked_run); if (psDBAffectedRows(config->dbh) < 1) { psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); @@ -1303,5 +1471,5 @@ query = pxDataGet("chiptool_change_exp_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) { psFree(query); // rollback @@ -1333,5 +1501,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } - +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) { @@ -1357,4 +1528,9 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); @@ -1370,5 +1546,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1406,5 +1582,5 @@ if (clean) { - bool success = true; + bool success = true; if (!strcmp(tables[i].tableName, "chipRun")) { success = pxSetStateCleaned("chipRun", "state", output); @@ -1448,4 +1624,14 @@ psVector *identifiers = psVectorAllocEmpty(16, PS_TYPE_U64); // Identifiers inserted + psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL); // Iterator + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + // first item is the dbversion, skip it + psMetadataItem *dbversion = psMetadataGetAndIncrement(iter); + (void) dbversion; + if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1453,5 +1639,4 @@ } - psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL); // Iterator psMetadataItem *item; // Item from iteration while ((item = psMetadataGetAndIncrement(iter))) { @@ -1482,2 +1667,205 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("chiptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +// a very specfic function to queue a cleaned chipProcessedImfile to be updated +static bool setimfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("chiptool_setimfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , chipRun.label = '%s'", label); + } + + if (class_id) { + psStringAppend(&query, " AND (chipProcessedImfile.class_id = '%s')", class_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "chipRun.dist_group", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipRun"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + psString query = pxDataGet("chiptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); + psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || where2) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, chip_id DESC"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/chiptool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/chiptool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/chiptool.h (revision 27840) @@ -26,4 +26,5 @@ CHIPTOOL_MODE_NONE = 0x0, CHIPTOOL_MODE_DEFINEBYQUERY, + CHIPTOOL_MODE_DEFINECOPY, CHIPTOOL_MODE_UPDATERUN, CHIPTOOL_MODE_PENDINGIMFILE, @@ -32,4 +33,5 @@ CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, + CHIPTOOL_MODE_SETIMFILETOUPDATE, CHIPTOOL_MODE_ADVANCEEXP, CHIPTOOL_MODE_BLOCK, @@ -45,6 +47,9 @@ CHIPTOOL_MODE_TOFULLIMFILE, CHIPTOOL_MODE_TOPURGEDIMFILE, + CHIPTOOL_MODE_TOSCRUBBEDIMFILE, CHIPTOOL_MODE_EXPORTRUN, - CHIPTOOL_MODE_IMPORTRUN + CHIPTOOL_MODE_IMPORTRUN, + CHIPTOOL_MODE_RUNSTATE, + CHIPTOOL_MODE_LISTRUN } chiptoolMode; Index: branches/simtest_nebulous_branches/ippTools/src/chiptoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/chiptoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/chiptoolConfig.c (revision 27840) @@ -39,5 +39,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -58,7 +58,29 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define distribution group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -definecopy + psMetadata *definecopyArgs = psMetadataAlloc(); + pxchipSetSearchArgs (definecopyArgs); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by rawExp label (LIKE comparison)", NULL); + + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); + + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_expgroup", 0, "define exposure group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define distribution group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updaterun @@ -67,8 +89,18 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "set data group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "set dist group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "set note", NULL); + + // -setimfiletoupdate + psMetadata *setimfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setimfiletoupdateArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id (required)", NULL); + psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0,"new value for label", NULL); // -pendingimfile @@ -148,5 +180,5 @@ psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); - psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", false); + psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", 0); // -processedimfile @@ -158,9 +190,14 @@ psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked status", false); + psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL); + pxmagicAddArguments(processedimfileArgs); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-allfiles", 0, "only all matching imfiles regardless of fault status", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + pxspaceAddArguments(processedimfileArgs); // -revertprocessedimfile @@ -179,4 +216,22 @@ psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + + // -processedExp + psMetadata *listrunArgs = psMetadataAlloc(); + pxchipSetSearchArgs(listrunArgs); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by chipRun state", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by chipRun dist_group (LIKE comparison)", NULL); + pxmagicAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + pxspaceAddArguments(listrunArgs); // -block @@ -226,4 +281,7 @@ psMetadataAddU64(runArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddStr(runArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + pxchipSetSearchArgs(runArgs); + psMetadataAddS64(runArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); // -advanceexp @@ -247,4 +305,9 @@ psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); // -exportrun @@ -260,9 +323,22 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", CHIPTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definecopy", "create copy runs", CHIPTOOL_MODE_DEFINECOPY, definecopyArgs); PXOPT_ADD_MODE("-updaterun", "change chip run properties", CHIPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-setimfiletoupdate", "set imfile state to update", CHIPTOOL_MODE_SETIMFILETOUPDATE, setimfiletoupdateArgs); PXOPT_ADD_MODE("-pendingimfile", "show pending imfiles", CHIPTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); PXOPT_ADD_MODE("-addprocessedimfile", "add a processed imfile", CHIPTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); @@ -270,4 +346,6 @@ PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs); PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); + PXOPT_ADD_MODE("-runstate", "list the states of chip run", CHIPTOOL_MODE_RUNSTATE, runstateArgs); + PXOPT_ADD_MODE("-listrun", "list chipRuns", CHIPTOOL_MODE_LISTRUN, listrunArgs); PXOPT_ADD_MODE("-advanceexp", "advance completed exposures", CHIPTOOL_MODE_ADVANCEEXP, advanceexpArgs); PXOPT_ADD_MODE("-block", "set a label block", CHIPTOOL_MODE_BLOCK, blockArgs); @@ -282,4 +360,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", CHIPTOOL_MODE_EXPORTRUN, exportrunArgs); @@ -287,5 +366,5 @@ if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { - psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); psFree(argSets); psFree(modes); Index: branches/simtest_nebulous_branches/ippTools/src/detselect.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/detselect.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/detselect.c (revision 27840) @@ -135,69 +135,5 @@ psString query = pxDataGet("detselect_search.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // we choose the single detrend image which matches all criteria and has - // the latest insertion date - - // unless explicitly specified by the user, list all possible matches - if (!unlimit) { - psStringAppend(&query, " ORDER BY registered DESC LIMIT 1"); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - // XXX check psError here - psError(PS_ERR_UNKNOWN, false, "no detrend exposures found"); - psFree(output); - return true; - } - - // negate simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "detExp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - -static bool selectMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); - PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - - psString query = pxDataGet("detselect_select.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -211,4 +147,68 @@ psFree(where); + // we choose the single detrend image which matches all criteria and has + // the latest insertion date + + // unless explicitly specified by the user, list all possible matches + if (!unlimit) { + psStringAppend(&query, " ORDER BY registered DESC LIMIT 1"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detrend exposures found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool selectMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString query = pxDataGet("detselect_select.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/simtest_nebulous_branches/ippTools/src/dettool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool.c (revision 27840) @@ -94,4 +94,5 @@ MODECASE(DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, pendingcleanup_processedexpMode); MODECASE(DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, donecleanup_processedexpMode); + MODECASE(DETTOOL_MODE_UPDATESTATE_PROCESSED, updatestateprocessedMode); // stacked MODECASE(DETTOOL_MODE_TOSTACKED, tostackedMode); @@ -142,4 +143,5 @@ MODECASE(DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, pendingcleanup_residexpMode); MODECASE(DETTOOL_MODE_DONECLEANUP_RESIDEXP, donecleanup_residexpMode); + MODECASE(DETTOOL_MODE_UPDATESTATE_RESID, updatestateresidMode); // detrunsummary MODECASE(DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryMode); @@ -150,4 +152,5 @@ MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); MODECASE(DETTOOL_MODE_RERUN, rerunMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryMode); // register MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); @@ -184,8 +187,8 @@ psString query = pxDataGet("dettool_pending.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); @@ -257,10 +260,10 @@ psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); return false; - } + } if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); return false; - } - + } + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); @@ -307,5 +310,5 @@ psMetadataItem *mItem = NULL; while ((mItem = psListGetAndIncrement(iter))) { - psS64 exp_id = mItem->data.S64; + psS64 exp_id = mItem->data.S64; if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); @@ -316,13 +319,13 @@ } psFree(iter); - } + } if (item->type == PS_DATA_S64) { - psS64 exp_id = item->data.S64; - if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); - psFree(where); - return false; - } - } + psS64 exp_id = item->data.S64; + if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } if (psListLength(where->list) < 1) { @@ -504,9 +507,9 @@ psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); return false; - } + } if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); return false; - } + } PXOPT_LOOKUP_STR(filelevel, config->args, "-filelevel", false, false); @@ -549,4 +552,5 @@ PXOPT_COPY_F32(config->args, where, "-select_airmass_max", "airmass", "<="); PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_max", "sat_pixel_frac", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sat_pixel_frac_min", "sat_pixel_frac", ">="); PXOPT_COPY_F32(config->args, where, "-select_exp_time_min", "exp_time", ">="); PXOPT_COPY_F32(config->args, where, "-select_exp_time_max", "exp_time", "<="); @@ -672,5 +676,5 @@ label, ref_det_id, - ref_iter + ref_iter ); psS64 det_id = psDBLastInsertID(config->dbh); @@ -741,5 +745,5 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - + // XXX this mode is not well-tested: probably need to specify iteration here PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required @@ -758,9 +762,9 @@ psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter"); return false; - } + } if (strcmp(mode, "verify") && ((ref_det_id != 0) || (ref_iter != -1))) { psError(PS_ERR_UNKNOWN, false, "master mode cannot have -ref_det_id or -ref_iter set"); return false; - } + } // the new detRun may have different values for these limits: @@ -816,5 +820,5 @@ if (iteration < 0) { - iteration = detRun->iteration; + iteration = detRun->iteration; } @@ -976,5 +980,5 @@ psString query = pxDataGet("dettool_definebydetrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -988,5 +992,5 @@ if (only_accepted) { - psStringAppend(&query, " AND accept = 1"); + psStringAppend(&query, " AND accept = 1"); } @@ -1067,5 +1071,5 @@ psString query = pxDataGet("dettool_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1137,5 +1141,5 @@ psString query = pxDataGet("dettool_childlessrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -1211,5 +1215,5 @@ psString query = pxDataGet("dettool_input.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -1263,5 +1267,5 @@ psString query = pxDataGet("dettool_raw.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1337,5 +1341,5 @@ } -// used by updatedetrunMode +// used by updatedetrunMode bool startNewIteration(pxConfig *config, psS64 det_id) { @@ -1344,5 +1348,5 @@ psString query = pxDataGet("dettool_start_new_iteration.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1563,5 +1567,5 @@ } detInputExpRow *newInputExp = detInputExpRowAlloc( - det_id, + det_id, newIteration, inputExp->exp_id, @@ -1661,6 +1665,6 @@ solang_max, label, - ref_det_id, - ref_iter + ref_det_id, + ref_iter )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1712,5 +1716,5 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - + PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required @@ -1727,20 +1731,20 @@ if (!detRegisteredImfileInsert(config->dbh, - det_id, - 0, // the iteration is fixed at 0 - class_id, - uri, - bg, - bg_stdev, - bg_mean_stdev, - user_1, - user_2, - user_3, - user_4, - user_5, - path_base, - "full", - 0 // fault code - )) { + det_id, + 0, // the iteration is fixed at 0 + class_id, + uri, + bg, + bg_stdev, + bg_mean_stdev, + user_1, + user_2, + user_3, + user_4, + user_5, + path_base, + "full", + 0 // fault code + )) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -1827,4 +1831,15 @@ if (!strcmp(data_state, "drop")) return true; if (!strcmp(data_state, "register")) return true; + // These are valid data states, and are necessary for the cleanup to work correctly. + if (!strcmp(data_state, "full")) return true; + if (!strcmp(data_state, "goto_cleaned")) return true; + if (!strcmp(data_state, "goto_scrubbed")) return true; + if (!strcmp(data_state, "goto_purged")) return true; + if (!strcmp(data_state, "cleaned")) return true; + if (!strcmp(data_state, "scrubbed")) return true; + if (!strcmp(data_state, "purged")) return true; + if (!strcmp(data_state, "error_cleaned")) return true; + if (!strcmp(data_state, "error_scrubbed")) return true; + if (!strcmp(data_state, "error_purged")) return true; psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state); @@ -1846,47 +1861,113 @@ -bool setProcessedImfileDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *class_id, const char *data_state) +bool setProcessedImfileDataState(pxConfig *config, psMetadata *where, const char *data_state) { PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where, false); PS_ASSERT_PTR_NON_NULL(data_state, false); - PS_ASSERT_PTR_NON_NULL(class_id, false); if (!isValidDataState (data_state)) return false; - char *query = "UPDATE detProcessedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND exp_id = %" PRId64 - " AND class_id = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, exp_id, class_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", exp_id %" PRId64 ", class_id %s", - det_id, exp_id, class_id); - return false; - } - - return true; -} - -bool setProcessedExpDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *data_state) + psString query = psStringCopy("UPDATE detProcessedImfile SET data_state = '%s'"); + if (where && psListLength(where->list) > 0) { + psString whereClause = psDBGenerateWhereSQL(where,NULL); + psStringAppend(&query," %s",whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + return(true); +} + +bool setProcessedExpDataState(pxConfig *config, psMetadata *where, const char *data_state) { PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where, false); PS_ASSERT_PTR_NON_NULL(data_state, false); if (!isValidDataState (data_state)) return false; - char *query = "UPDATE detProcessedExp SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND exp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, exp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", exp_id %" PRId64, - det_id, exp_id); - return false; - } - - return true; -} - - + psString query = psStringCopy("UPDATE detProcessedExp SET data_state = '%s'"); + if (where && psListLength(where->list) > 0) { + psString whereClause = psDBGenerateWhereSQL(where,NULL); + psStringAppend(&query," %s",whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + return(true); +} + +bool setResidImfileDataState(pxConfig *config, psMetadata *where, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where,false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + psString query = psStringCopy("UPDATE detResidImfile SET data_state = '%s'"); + if (where && psListLength(where->list) > 0) { + psString whereClause = psDBGenerateWhereSQL(where,NULL); + psStringAppend(&query," %s",whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + return(true); +} +// This function apparently not used anymore. +bool setResidExpDataState(pxConfig *config, psMetadata *where, const char *data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where,false); + PS_ASSERT_PTR_NON_NULL(data_state, false); + + if (!isValidDataState (data_state)) return false; + + psString query = psStringCopy("UPDATE detResidExp SET data_state = '%s'"); + if (where && psListLength(where->list) > 0) { + psString whereClause = psDBGenerateWhereSQL(where,NULL); + psStringAppend(&query," %s",whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + return(true); +} + +// Not yet updated for cleaning bool setStackedImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state) { @@ -1898,11 +1979,11 @@ char *query = "UPDATE detStackedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND class_id = %s"; + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32 "class_id %s", - det_id, iteration, class_id); + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 "class_id %s", + det_id, iteration, class_id); return false; } @@ -1920,11 +2001,12 @@ char *query = "UPDATE detNormalizedStatImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND class_id = %s"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration)) { + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = '%s'"; +/* fprintf(stderr,"DETTOOL SAYS: %s\n",query); */ + if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration,class_id)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32, - det_id, iteration); + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, + det_id, iteration); return false; } @@ -1942,11 +2024,11 @@ char *query = "UPDATE detNormalizedImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND class_id = %s"; + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32 + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32 " class %s", - det_id, iteration, class_id); + "failed to change state for det_id %" PRId64 ", iteration %" PRId32 " class %s", + det_id, iteration, class_id); return false; } @@ -1963,10 +2045,10 @@ char *query = "UPDATE detNormalizedExp SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32; + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration)) { psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32, - det_id, iteration); + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, + det_id, iteration); return false; } @@ -1974,48 +2056,5 @@ return true; } - -bool setResidImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *class_id, const char *data_state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(data_state, false); - PS_ASSERT_PTR_NON_NULL(class_id, false); - - if (!isValidDataState (data_state)) return false; - - char *query = "UPDATE detResidImfile SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND exp_id = %" PRId64 - " AND class_id = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, exp_id, class_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32 ", exp_id %" PRId64 ", class_id %s", - det_id, iteration, exp_id, class_id); - return false; - } - - return true; -} - -bool setResidExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *data_state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(data_state, false); - - if (!isValidDataState (data_state)) return false; - - char *query = "UPDATE detResidExp SET data_state = '%s'" - " WHERE det_id = %" PRId64 - " AND iteration = %" PRId32 - " AND exp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, exp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for det_id %" PRId64 ", iteration %" PRId32 ", exp_id %" PRId64, - det_id, iteration, exp_id); - return false; - } - - return true; -} +// End not yet modified for cleanup. bool exportrunMode(pxConfig *config) @@ -2025,5 +2064,5 @@ char sqlFilename[80]; } ExportTable; - + int numExportTables = 12; @@ -2061,5 +2100,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -2103,5 +2142,5 @@ } psFree(output); - + } @@ -2114,7 +2153,7 @@ { unsigned int nFail; - + int numImportTables = 11; - + char tables[11] [80] = {"detInputExp", "detNormalizedExp", "detNormalizedImfile", "detNormalizedStatImfile", "detProcessedExp", @@ -2123,5 +2162,5 @@ PS_ASSERT_PTR_NON_NULL(config, NULL); - + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); @@ -2130,5 +2169,5 @@ fprintf (stdout, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - + psMetadataItem *item = psMetadataLookup (input, "detRun"); psAssert (item, "entry not in input?"); @@ -2140,5 +2179,5 @@ detRunRow *detRun = detRunObjectFromMetadata (entry->data.md); detRunInsertObject (config->dbh, detRun); - + // fprintf (stdout, "---- det run ----\n"); // psMetadataPrint (stderr, entry->data.md, 1); @@ -2148,5 +2187,5 @@ psAssert (item, "entry not in input?"); psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - + switch (i) { case 0: @@ -2162,5 +2201,5 @@ } break; - + case 1: for (int i = 0; i < item->data.list->n; i++) { @@ -2175,5 +2214,5 @@ } break; - + case 2: for (int i = 0; i < item->data.list->n; i++) { @@ -2188,5 +2227,5 @@ } break; - + case 3: for (int i = 0; i < item->data.list->n; i++) { @@ -2201,5 +2240,5 @@ } break; - + case 4: for (int i = 0; i < item->data.list->n; i++) { @@ -2214,5 +2253,5 @@ } break; - + case 5: for (int i = 0; i < item->data.list->n; i++) { @@ -2227,5 +2266,5 @@ } break; - + case 6: for (int i = 0; i < item->data.list->n; i++) { @@ -2240,5 +2279,5 @@ } break; - + case 7: for (int i = 0; i < item->data.list->n; i++) { @@ -2253,5 +2292,5 @@ } break; - + case 8: for (int i = 0; i < item->data.list->n; i++) { @@ -2266,5 +2305,5 @@ } break; - + case 9: for (int i = 0; i < item->data.list->n; i++) { @@ -2279,5 +2318,5 @@ } break; - + case 10: for (int i = 0; i < item->data.list->n; i++) { Index: branches/simtest_nebulous_branches/ippTools/src/dettool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool.h (revision 27840) @@ -57,4 +57,5 @@ DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, + DETTOOL_MODE_UPDATESTATE_PROCESSED, DETTOOL_MODE_TOSTACKED, @@ -105,4 +106,5 @@ DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, DETTOOL_MODE_DONECLEANUP_RESIDEXP, + DETTOOL_MODE_UPDATESTATE_RESID, DETTOOL_MODE_TODETRUNSUMMARY, @@ -111,4 +113,5 @@ DETTOOL_MODE_REVERTDETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUNSUMMARY, + DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUN, DETTOOL_MODE_RERUN, @@ -148,4 +151,5 @@ bool pendingcleanup_processedexpMode(pxConfig *config); bool donecleanup_processedexpMode(pxConfig *config); +bool updatestateprocessedMode(pxConfig *config); // stackedimfile @@ -202,4 +206,5 @@ bool pendingcleanup_residexpMode(pxConfig *config); bool donecleanup_residexpMode(pxConfig *config); +bool updatestateresidMode(pxConfig *config); // detrunsummary @@ -209,4 +214,5 @@ bool revertdetrunsummaryMode(pxConfig *config); bool updatedetrunsummaryMode(pxConfig *config); +bool pendingcleanup_detrunsummaryMode(pxConfig *config); // other utilities @@ -218,12 +224,13 @@ // functions to set the 'data_state' for stages -bool setProcessedImfileDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *class_id, const char *data_state); -bool setProcessedExpDataState(pxConfig *config, psS64 det_id, psS64 exp_id, const char *data_state); +bool setProcessedImfileDataState(pxConfig *config, psMetadata *where, const char *data_state); +bool setProcessedExpDataState(pxConfig *config, psMetadata *where, const char *data_state); +bool setResidImfileDataState(pxConfig *config, psMetadata *where, const char *data_state); +bool setResidExpDataState(pxConfig *config, psMetadata *where, const char *data_state); + bool setStackedImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); bool setNormStatImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); bool setNormImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *class_id, const char *data_state); bool setNormExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, const char *data_state); -bool setResidImfileDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *class_id, const char *data_state); -bool setResidExpDataState(pxConfig *config, psS64 det_id, psS32 iteration, psS64 exp_id, const char *data_state); #endif // DETTOOL_H Index: branches/simtest_nebulous_branches/ippTools/src/dettoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettoolConfig.c (revision 27840) @@ -31,5 +31,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -41,11 +41,11 @@ // XXX EAM : is this used? does it make sense? psMetadata *pendingArgs = psMetadataAlloc(); - psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, "search by URL", NULL); - psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definebytag @@ -117,4 +117,5 @@ psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_airmass_max", 0, "define max airmass", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_sat_pixel_frac_min", 0, "define min fraction of saturated pixels", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_min", 0, "define min exposure time", NAN); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_time_max", 0, "define max exposure time", NAN); @@ -125,4 +126,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); @@ -137,4 +139,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max", 0, "define max moon phase", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); @@ -207,7 +210,7 @@ // -addcorrectimfile psMetadata *addcorrectimfileArgs = psMetadataAlloc(); - psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); @@ -303,8 +306,8 @@ // -updateprocessedimfile psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedimfile @@ -312,5 +315,5 @@ psMetadataAddBool(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(pendingcleanup_processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -320,5 +323,5 @@ psMetadataAddBool(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(donecleanup_processedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -364,7 +367,7 @@ // -updateprocessedexp psMetadata *updateprocessedexpArgs = psMetadataAlloc(); - psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); - psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedexp @@ -372,5 +375,5 @@ psMetadataAddBool(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(pendingcleanup_processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); @@ -379,6 +382,12 @@ psMetadataAddBool(donecleanup_processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS64(donecleanup_processedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + + // -updatestate_processed + psMetadata *updatestateprocessedArgs = psMetadataAlloc(); + psMetadataAddS64(updatestateprocessedArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); + psMetadataAddS64(updatestateprocessedArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updatestateprocessedArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -tostacked @@ -423,8 +432,8 @@ // -updatestacked psMetadata *updatestackedArgs = psMetadataAlloc(); - psMetadataAddS64(updatestackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updatestackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); - psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_stacked @@ -432,5 +441,5 @@ psMetadataAddBool(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(pendingcleanup_stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -440,5 +449,5 @@ psMetadataAddBool(donecleanup_stackedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_stackedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_stackedArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_stackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(donecleanup_stackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); @@ -475,8 +484,8 @@ // -updatenormalizedstat psMetadata *updatenormalizedstatArgs = psMetadataAlloc(); - psMetadataAddS64(updatenormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updatenormalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedstat @@ -484,5 +493,5 @@ psMetadataAddBool(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(pendingcleanup_normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -492,5 +501,5 @@ psMetadataAddBool(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(donecleanup_normalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -537,8 +546,8 @@ // -updatenormalizedimfile psMetadata *updatenormalizedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(updatenormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updatenormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedimfile @@ -546,5 +555,5 @@ psMetadataAddBool(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(pendingcleanup_normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -554,5 +563,5 @@ psMetadataAddBool(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(donecleanup_normalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); @@ -596,7 +605,7 @@ // -updatenormalizedexp psMetadata *updatenormalizedexpArgs = psMetadataAlloc(); - psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); - psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedexp @@ -604,5 +613,5 @@ psMetadataAddBool(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); @@ -611,5 +620,5 @@ psMetadataAddBool(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_normalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); @@ -674,9 +683,9 @@ // -updateresidimfile psMetadata *updateresidimfileArgs = psMetadataAlloc(); - psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(updateresidimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residimfile @@ -684,5 +693,5 @@ psMetadataAddBool(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(pendingcleanup_residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); @@ -693,5 +702,5 @@ psMetadataAddBool(donecleanup_residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); + psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); psMetadataAddS32(donecleanup_residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(donecleanup_residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); @@ -759,5 +768,5 @@ psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); - psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residexp @@ -776,4 +785,10 @@ psMetadataAddS32(donecleanup_residexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddS64(donecleanup_residexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + + // -updatestate_resid + psMetadata *updatestateresidArgs = psMetadataAlloc(); + psMetadataAddS64(updatestateresidArgs, PS_LIST_TAIL, "-det_id", 0, "search by det ID", 0); + psMetadataAddS64(updatestateresidArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updatestateresidArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -todetrunsummary @@ -813,4 +828,13 @@ psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "reject this detrun iteration as a master", false); + psMetadataAddStr(updatedetrunsummaryArgs, PS_LIST_TAIL, "-data_state", 0, "set the data state", NULL); + psMetadataAddS32(updatedetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -pendingcleanup_detrunsummary + psMetadata *pendingcleanup_detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddU64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updatedetrun @@ -913,4 +937,5 @@ PXOPT_ADD_MODE("-pendingcleanup_processedexp", "", DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDEXP, pendingcleanup_processedexpArgs); PXOPT_ADD_MODE("-donecleanup_processedexp", "", DETTOOL_MODE_DONECLEANUP_PROCESSEDEXP, donecleanup_processedexpArgs); + PXOPT_ADD_MODE("-updatestate_processed", "", DETTOOL_MODE_UPDATESTATE_PROCESSED, updatestateprocessedArgs); PXOPT_ADD_MODE("-tostacked", "", DETTOOL_MODE_TOSTACKED, tostackedArgs); @@ -961,4 +986,5 @@ PXOPT_ADD_MODE("-pendingcleanup_residexp", "", DETTOOL_MODE_PENDINGCLEANUP_RESIDEXP, pendingcleanup_residexpArgs); PXOPT_ADD_MODE("-donecleanup_residexp", "", DETTOOL_MODE_DONECLEANUP_RESIDEXP, donecleanup_residexpArgs); + PXOPT_ADD_MODE("-updatestate_resid", "", DETTOOL_MODE_UPDATESTATE_RESID, updatestateresidArgs); PXOPT_ADD_MODE("-todetrunsummary", "", DETTOOL_MODE_TODETRUNSUMMARY, todetrunsummaryArgs); @@ -967,4 +993,6 @@ PXOPT_ADD_MODE("-revertdetrunsummary", "", DETTOOL_MODE_REVERTDETRUNSUMMARY, revertdetrunsummaryArgs); PXOPT_ADD_MODE("-updatedetrunsummary", "", DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryArgs); + + PXOPT_ADD_MODE("-pendingcleanup_detrunsummary", "", DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryArgs); PXOPT_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); Index: branches/simtest_nebulous_branches/ippTools/src/dettool_correction.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_correction.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_correction.c (revision 27840) @@ -34,5 +34,5 @@ PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - // build the needed where + // build the needed where psMetadata *where = psMetadataAlloc(); psMetadataAddS64(where, PS_LIST_TAIL, "det_id", 0, "==", ref_det_id); @@ -88,6 +88,6 @@ detRun->solang_max, use_label, - detRun->det_id, // ref_det_id - detRun->iteration // ref_iter + detRun->det_id, // ref_det_id + detRun->iteration // ref_iter ); psFree(runs); @@ -140,11 +140,11 @@ psString query = pxDataGet("dettool_tocorrectimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } if (psListLength(where->list)) { - // NOTE the SQL uses an intermediate table 'det1' for this query - psString whereClause = psDBGenerateWhereConditionSQL(where, "det1"); + // NOTE the SQL uses an intermediate table 'det1' for this query + psString whereClause = psDBGenerateWhereConditionSQL(where, "det1"); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -198,5 +198,5 @@ psString query = pxDataGet("dettool_tocorrectexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -248,9 +248,9 @@ return false; } - + // automatically stop completed 'correct' detRuns psString query = pxDataGet("dettool_stop_completed_correct_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_detrunsummary.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_detrunsummary.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_detrunsummary.c (revision 27840) @@ -35,5 +35,5 @@ psString query = pxDataGet("dettool_todetrunsummary.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -135,5 +135,5 @@ psString query = pxDataGet("dettool_find_completed_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -241,5 +241,5 @@ psString query = pxDataGet("dettool_detrunsummary.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -310,5 +310,5 @@ psString query = pxDataGet("dettool_revertdetrunsummary.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -336,5 +336,65 @@ } -// XXX need to add -data_state here +bool pendingcleanup_detrunsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_detrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + // fprintf(stderr,">>>%s<<<\n",query); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingCleanupDetRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return(false); + } + psFree(output); + + return(true); +} + + +// preliminary code now. bool updatedetrunsummaryMode(pxConfig *config) { @@ -344,4 +404,7 @@ PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", ((accept == 0)&&(reject == 0)), false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", ((accept == 0)&&(reject == 0)), false); + if (accept && reject) { @@ -350,13 +413,109 @@ } - if (!(accept || reject)) { - psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required"); - return false; - } - - char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; - if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + if (!(accept || reject || (data_state != NULL))) { + psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required if -data_state is not supplied"); + return false; + } + + if (accept || reject) { + char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + else { + PS_ASSERT_PTR_NON_NULL(data_state,false); + +/* if (!isValidDataState(data_state)) return false; */ + + + + char *query_detRunSummary = "UPDATE detRunSummary SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detRunSummary, data_state, det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return false; + } + + /* This check allows the one update to flag everything for cleanup. The check for full is only temporary while I test for bugs. */ + if ((!strncmp(data_state,"goto_",5) + // || (!strcmp(data_state,"full")) + // || (!strcmp(data_state,"cleaned")) + )) { + char *query_detProcessedImfile = "UPDATE detProcessedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedImfile,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detProcessedExp = "UPDATE detProcessedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedExp,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedImfile = "UPDATE detNormalizedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedStatImfile = "UPDATE detNormalizedStatImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedStatImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedStatImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedExp = "UPDATE detNormalizedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidImfile = "UPDATE detResidImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidExp = "UPDATE detResidExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detStackedImfile = "UPDATE detStackedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detStackedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detStackedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + } + /* End if */ + } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedexp.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedexp.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedexp.c (revision 27840) @@ -29,5 +29,5 @@ psString query = pxDataGet("dettool_tonormalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -89,5 +89,5 @@ psString query = pxDataGet("dettool_tonormalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -156,5 +156,5 @@ psString query = pxDataGet("dettool_normalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -225,5 +225,5 @@ psString query = pxDataGet("dettool_revertnormalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -278,5 +278,5 @@ psString query = pxDataGet("dettool_pendingcleanup_normalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -340,5 +340,5 @@ psString query = pxDataGet("dettool_donecleanup_normalizedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedimfile.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedimfile.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedimfile.c (revision 27840) @@ -30,5 +30,5 @@ psString query = pxDataGet("dettool_tonormalize.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -137,5 +137,5 @@ psString query = pxDataGet("dettool_normalizedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -205,5 +205,5 @@ psString query = pxDataGet("dettool_revertnormalizedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -260,5 +260,5 @@ psString query = pxDataGet("dettool_pendingcleanup_normalizedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -323,5 +323,5 @@ psString query = pxDataGet("dettool_donecleanup_normalizedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedstat.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedstat.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_normalizedstat.c (revision 27840) @@ -29,5 +29,5 @@ psString query = pxDataGet("dettool_tonormalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -117,5 +117,5 @@ psString query = pxDataGet("dettool_normalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -185,5 +185,5 @@ psString query = pxDataGet("dettool_revertnormalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -240,5 +240,5 @@ psString query = pxDataGet("dettool_pendingcleanup_normalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -303,5 +303,5 @@ psString query = pxDataGet("dettool_donecleanup_normalizedstat.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_processedexp.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_processedexp.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_processedexp.c (revision 27840) @@ -29,5 +29,5 @@ psString query = pxDataGet("dettool_toprocessedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -101,5 +101,5 @@ psString query = pxDataGet("dettool_addprocessedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -235,5 +235,5 @@ psString query = pxDataGet("dettool_revertprocessedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -266,10 +266,36 @@ PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); - if (!setProcessedExpDataState(config, det_id, exp_id, data_state)) { - return false; - } - return true; + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id","det_id","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id","exp_id","=="); + + if (!setProcessedExpDataState(config, where, data_state)) { + return false; + } + psFree(where); + return true; +} + +bool updatestateprocessedMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + PXOPT_LOOKUP_S64(det_id,config->args, "-det_id",true,false); + PXOPT_LOOKUP_STR(data_state,config->args, "-data_state", true, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id","det_id","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id","exp_id","=="); + + if (!setProcessedExpDataState(config,where,data_state)) { + return(false); + } + if (!setProcessedImfileDataState(config,where,data_state)) { + return(false); + } + psFree(where); + return(true); } @@ -287,5 +313,5 @@ psString query = pxDataGet("dettool_pendingcleanup_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -349,5 +375,5 @@ psString query = pxDataGet("dettool_donecleanup_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_processedimfile.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_processedimfile.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_processedimfile.c (revision 27840) @@ -34,5 +34,5 @@ psString query = pxDataGet("dettool_toprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -180,5 +180,5 @@ psString query = pxDataGet("dettool_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -267,5 +267,5 @@ psString query = pxDataGet("dettool_revertprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -298,11 +298,16 @@ PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); - if (!setProcessedImfileDataState(config, det_id, exp_id, class_id, data_state)) { - return false; - } + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id", "det_id","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id", "exp_id","=="); + PXOPT_COPY_STR(config->args,where,"-class_id","class_id","=="); + + if (!setProcessedImfileDataState(config, where, data_state)) { + return false; + } + psFree(where); return true; } @@ -314,5 +319,5 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - +/* int i; */ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); @@ -322,5 +327,5 @@ psString query = pxDataGet("dettool_pendingcleanup_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -345,4 +350,5 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",query); */ psFree(query); @@ -350,8 +356,10 @@ if (!output) { psError(PS_ERR_UNKNOWN, false, "database error"); +/* fprintf(stderr,"WTF !output?\n"); */ return false; } if (!psArrayLength(output)) { psTrace("dettool", PS_LOG_INFO, "no rows found"); +/* fprintf(stderr,"WTF no rows??\n"); */ psFree(output); return true; @@ -359,4 +367,6 @@ // negative simple so the default is true +/* i = (int) ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple); */ +/* fprintf(stderr,">>%d<<\n",i); */ if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -364,4 +374,6 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",output); */ +/* psFree(output); */ psFree(output); @@ -385,5 +397,5 @@ psString query = pxDataGet("dettool_donecleanup_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_residexp.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_residexp.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_residexp.c (revision 27840) @@ -35,5 +35,5 @@ psString query = pxDataGet("dettool_toresidexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -119,5 +119,5 @@ psString query = pxDataGet("dettool_toresidexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -202,5 +202,5 @@ psString query = pxDataGet("dettool_residexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -277,5 +277,5 @@ psString query = pxDataGet("dettool_revertresidexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -353,4 +353,24 @@ } +bool updatestateresidMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + PXOPT_LOOKUP_S64(det_id,config->args, "-det_id",true,false); + PXOPT_LOOKUP_STR(data_state,config->args, "-data_state", true, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id","det_id","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id","exp_id","=="); + + if (!setResidExpDataState(config,where,data_state)) { + return(false); + } + if (!setResidImfileDataState(config,where,data_state)) { + return(false); + } + psFree(where); + return(true); +} + bool pendingcleanup_residexpMode(pxConfig *config) { @@ -367,5 +387,5 @@ psString query = pxDataGet("dettool_pendingcleanup_residexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -430,5 +450,5 @@ psString query = pxDataGet("dettool_donecleanup_residexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_residimfile.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_residimfile.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_residimfile.c (revision 27840) @@ -29,5 +29,5 @@ psString query = pxDataGet("dettool_toresidimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -165,5 +165,5 @@ psString query = pxDataGet("dettool_residimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -253,5 +253,5 @@ psString query = pxDataGet("dettool_revertresidimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -284,9 +284,14 @@ PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); - PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", true, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); - PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false); - if (!setResidImfileDataState(config, det_id, iteration, exp_id, class_id, data_state)) { + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args,where,"-det_id","det_id","=="); + PXOPT_COPY_S32(config->args,where,"-iteration", "iteration","=="); + PXOPT_COPY_S64(config->args,where,"-exp_id","exp_id","=="); + PXOPT_COPY_STR(config->args,where,"-class_id","class_id","=="); + + if (!setResidImfileDataState(config, where, data_state)) { return false; } @@ -309,5 +314,5 @@ psString query = pxDataGet("dettool_pendingcleanup_residimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -373,5 +378,5 @@ psString query = pxDataGet("dettool_donecleanup_residimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/dettool_stack.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dettool_stack.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/dettool_stack.c (revision 27840) @@ -31,5 +31,5 @@ psString query = pxDataGet("dettool_tostacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -220,5 +220,5 @@ psString query = pxDataGet("dettool_stacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -290,5 +290,5 @@ psString query = pxDataGet("dettool_revertstacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -345,5 +345,5 @@ psString query = pxDataGet("dettool_pendingcleanup_stacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -408,5 +408,5 @@ psString query = pxDataGet("dettool_donecleanup_stacked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/difftool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/difftool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/difftool.c (revision 27840) @@ -37,4 +37,5 @@ static bool todiffskyfileMode(pxConfig *config); static bool adddiffskyfileMode(pxConfig *config); +static bool advanceMode(pxConfig *config); static bool diffskyfileMode(pxConfig *config); static bool revertdiffskyfileMode(pxConfig *config); @@ -42,4 +43,5 @@ static bool definewarpstackMode(pxConfig *config); static bool definewarpwarpMode(pxConfig *config); +static bool definestackstackMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -50,5 +52,13 @@ static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); -static bool diffRunComplete(pxConfig *config); +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state); +static bool tocleanedskyfileMode(pxConfig *config); +static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); +static bool tofullskyfileMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); + + # define MODECASE(caseName, func) \ @@ -76,9 +86,11 @@ MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileMode); MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileMode); + MODECASE(DIFFTOOL_MODE_ADVANCE, advanceMode); MODECASE(DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileMode); MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); - MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); + MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); + MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -87,4 +99,11 @@ MODECASE(DIFFTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(DIFFTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); + MODECASE(DIFFTOOL_MODE_LISTRUN, listrunMode); + MODECASE(DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); + default: psAbort("invalid option (this should not happen)"); @@ -114,10 +133,14 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false); PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_S16(diff_mode, config->args, "-set_diff_mode", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -130,4 +153,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -136,5 +161,7 @@ bothways, exposure, - false + false, + diff_mode, + note ); if (!run) { @@ -167,14 +194,25 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required options - PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { - // set detRun.state to state - return setdiffRunState(config, diff_id, state, false); - } - - return true; + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group","data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE diffRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "diffRun", "diff_id", "diffSkyfile", true); + + psFree(query); + psFree(where); + + return result; } @@ -331,27 +369,23 @@ psString query = pxDataGet("difftool_inputskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString whereClause = NULL; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psStringCopy(""); if (psListLength(where->list)) { whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&whereClause, "\n AND "); + psStringPrepend(&whereClause, "\n WHERE "); } psFree(where); // Add condition to get only templates or only inputs + psString templateClause = psStringCopy(""); { - psString templateClause = NULL; if (template) { - psStringAppend(&templateClause, " %s", " template != 0"); + psStringAppend(&templateClause, "\n WHERE %s", " template != 0"); } else if (input) { - psStringAppend(&templateClause, " %s", " template = 0"); - } - if (templateClause) { - psStringAppend(&whereClause, "\n AND %s", templateClause); - } - psFree(templateClause); + psStringAppend(&templateClause, "\n WHERE %s", " template = 0"); + } } @@ -364,10 +398,12 @@ } - if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause, whereClause, templateClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(templateClause); psFree(whereClause); psFree(query); return false; } + psFree(templateClause); psFree(whereClause); psFree(query); @@ -421,5 +457,5 @@ psString query = pxDataGet("difftool_todiffskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -555,12 +591,4 @@ } - if (!diffRunComplete(config)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - // point of no return if (!psDBCommit(config->dbh)) { @@ -569,4 +597,95 @@ } + + return true; +} + +static bool advanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // look for completed diffRuns + psString query = pxDataGet("difftool_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(whereString); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + psFree(whereString); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); + psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + + // set diffRun.state to 'full' + if (!setdiffRunState(config, diff_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -581,17 +700,40 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); - PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-warp_id", "warpRun.warp_id", "=="); - + PXOPT_COPY_S64(config->args, where, "-magicked", "diffSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + if (!template) { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "LIKE"); + } else { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "LIKE"); + } + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "diffSkyfile"); + pxspaceAddWhere(config, &where2, template ? "rawTemplate" : "rawInput"); psString query = pxDataGet("difftool_skyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -601,7 +743,20 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (where2) { + psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; } psFree(where); + if (pstamp_order) { + if (template) { + psStringAppend(&query, " ORDER BY rawTemplate.exp_id, diff_id DESC"); + } else { + psStringAppend(&query, " ORDER BY rawInput.exp_id, diff_id DESC"); + } + } + // treat limit == 0 as "no limit" if (limit) { @@ -659,27 +814,23 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "diffRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "diffRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Update state to 'new' + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + { - psString query = pxDataGet("difftool_revertdiffskyfile_update.sql"); + psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString query_updated = pxDataGet("difftool_revertdiffskyfile_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -688,4 +839,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); psFree(whereClause); } @@ -694,50 +846,21 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } return false; } psFree(query); + psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); } - // Delete product - { - psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); - } - psFree(where); - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } return true; @@ -755,14 +878,47 @@ } - char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { + if (magicked) { + char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for diff_id %"PRId64, diff_id); return false; + } + } + else { + char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, diff_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for diff_id %"PRId64, diff_id); + return false; + } } return true; } + + +#ifdef notdef +static bool setdiffRunStateByLabel(pxConfig *config, const char *label, const char *state) { + PS_ASSERT_PTR_NON_NULL(state,false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid diffRun state: %s", state); + return false; + } + + char *query = "UPDATE diffRun SET state = '%s' WHERE label = '%s'"; + if (!p_psDBRunQueryF(config->dbh,query,state,label)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for label %s", label); + return(false); + } + + return true; +} +#endif // Generate a single populated run @@ -772,5 +928,8 @@ const char *tess_id, // Tessellation identifier const char *label, // label + const char *data_group, // data_group + const char *dist_group, // dist_group const char *reduction, // reduction + const char *note, // note psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none @@ -793,4 +952,18 @@ return false; } + psS16 diff_mode = IPP_DIFF_MODE_UNDEFINED; + if ((input_warp_id != PS_MAX_S64) && (template_warp_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_WARP_WARP; + } + else if ((input_warp_id != PS_MAX_S64) && (template_stack_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_WARP_STACK; + } + else if ((input_stack_id != PS_MAX_S64) && (template_warp_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_STACK_WARP; + } + else if ((input_stack_id != PS_MAX_S64) && (template_stack_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_STACK_STACK; + } + // default @@ -803,4 +976,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -809,5 +984,7 @@ false, false, - 0 // magicked + 0, // magicked + diff_mode, // diff_mode + note ); @@ -883,4 +1060,6 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required options PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); PXOPT_LOOKUP_S64(template_warp_id, config->args, "-template_warp_id", false, false); @@ -889,4 +1068,5 @@ PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); if (template_stack_id && template_warp_id) { @@ -911,6 +1091,6 @@ psArray *list = psArrayAllocEmpty(16); // List of runs, to print - - if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, + // Populated Run will generate the diff_mode value + if (!populatedrun(list, workdir, skycell_id, tess_id, label, data_group ? data_group : label, dist_group, reduction, note, input_warp_id ? input_warp_id : PS_MAX_S64, input_stack_id ? input_stack_id : PS_MAX_S64, @@ -939,20 +1119,31 @@ psMetadata *expWhere = psMetadataAlloc(); - psMetadata *warpWhere = psMetadataAlloc(); + psMetadata *warp1Where = psMetadataAlloc(); // First set of restrictions on warp + psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp psMetadata *stackWhere = psMetadataAlloc(); PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "=="); PXOPT_COPY_STR(config->args, expWhere, "-filter", "filter", "=="); - PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpSkyfile.skycell_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-tess_id", "warpRun.tess_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "=="); - PXOPT_COPY_F32(config->args, warpWhere, "-good_frac", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, expWhere, "-comment", "comment", "LIKE"); + PXOPT_COPY_S64(config->args, warp1Where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-warp_label", "warpRun.label", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "=="); + PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "=="); + PXOPT_COPY_F32(config->args, warp2Where, "-good_frac", "warpSkyfile.good_frac", ">="); PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "=="); - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); @@ -964,9 +1155,10 @@ psString query = pxDataGet("difftool_definewarpstack_part1.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString warpQuery = NULL; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString warp1Query = NULL; + psString warp2Query = NULL; psString stackQuery = NULL; psString expQuery = NULL; @@ -980,17 +1172,20 @@ } psFree(expWhere); - if (psListLength(warpWhere->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL); - psStringAppend(&warpQuery, "\n AND %s", whereClause); + if (psListLength(warp1Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(warp1Where, NULL); + psStringAppend(&warp1Query, "\n AND %s", whereClause); psFree(whereClause); } else { - warpQuery = psStringCopy("\n"); - } - psFree(warpWhere); - - if (!available) { - // diff what's available, even if warp run has some faults and is incomplete - psStringAppend(&warpQuery, " AND warpRun.state = 'full'"); - } + warp1Query = psStringCopy("\n"); + } + psFree(warp1Where); + if (psListLength(warp2Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(warp2Where, NULL); + psStringAppend(&warp2Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + warp2Query = psStringCopy("\n"); + } + psFree(warp2Where); // don't queue for exposures that have already been diff'd unless requested @@ -1011,5 +1206,5 @@ psFree(stackWhere); - psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery); + psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery); if (!psDBTransaction(config->dbh)) { @@ -1018,5 +1213,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, warpQuery, expQuery, diffQuery)) { + if (!p_psDBRunQueryF(config->dbh, query, warp1Query, warp2Query, expQuery, diffQuery)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1078,4 +1273,5 @@ return false; } + psFree(warp1Query); psFree(query); query = NULL; @@ -1109,5 +1305,5 @@ if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1120,5 +1316,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1131,5 +1327,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "skycell_count not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1142,5 +1338,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "tess_id not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1150,8 +1346,9 @@ return false; } + psString filter = psMetadataLookupStr(&mdok, row, "filter"); if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "filter not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1161,7 +1358,23 @@ return false; } - if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warpQuery)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + + psString warp_data_group = psMetadataLookupStr(&mdok, row, "data_group"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "warp data_group not found"); + psFree(warp2Query); + psFree(stackQuery); + psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (warp_data_group) { + data_group = warp_data_group; + } + + if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warp2Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1174,11 +1387,11 @@ if (num == 0) { - psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id); + psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for warp_id %" PRId64, warp_id); continue; } if (!available && (num != skycell_count)) { - psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" - PRId64 " need %" PRId64, num, skycell_count, warp_id); + psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64 + " but need %" PRId64, num, warp_id, skycell_count); continue; } @@ -1190,4 +1403,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -1196,5 +1411,7 @@ false, // bothways true, // exposure - 0 // magicked + 0, // magicked + IPP_DIFF_MODE_WARP_STACK, + note ); @@ -1212,5 +1429,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1226,5 +1443,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1241,5 +1458,5 @@ psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, run->diff_id); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1255,5 +1472,5 @@ } psFree(output); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1265,5 +1482,5 @@ } - if (!diffRunPrintObjects(stdout, list, !simple)) { + if (numGood && !diffRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); psFree(list); @@ -1343,11 +1560,14 @@ // Additional controls PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); - PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); // Settings to apply to defined run - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required options + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); // option + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); // option + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -1364,5 +1584,5 @@ psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL if (!tempCreate) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1387,5 +1607,5 @@ psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table if (!tempInsert) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1419,5 +1639,5 @@ psString select = pxDataGet("difftool_definewarpwarp_select.sql"); if (!select) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1432,18 +1652,11 @@ if (psListLength(selectWhere->list)) { psString new = psDBGenerateWhereConditionSQL(selectWhere, NULL); - psStringAppend(&whereClause, "\n%s %s", whereClause ? "AND" : "WHERE", new); + psStringAppend(&whereClause, "\nAND %s", new); psFree(new); } psFree(selectWhere); - if (!available) { - psStringAppend(&whereClause, - "\n%s inputWarpRun.state = 'full'" - "AND templateWarpRun.state = 'full'", - whereClause ? "AND" : "WHERE"); - } - if (!rerun) { - psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE"); + psStringAppend(&whereClause, "\nAND diffs.diff_id IS NULL"); } @@ -1522,4 +1735,9 @@ const char *template = psMetadataLookupStr(NULL, row, "template_warp_id"); const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); + psString input_data_group = psMetadataLookupStr(NULL, row, "input_data_group"); + if (input_data_group) { + data_group = input_data_group; + } + if (!input_id || !template || !tess_id) { psError(PXTOOLS_ERR_PROG, false, "Identifiers not found"); @@ -1532,7 +1750,20 @@ return false; } - - diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered, - tess_id, true, true, false); // Run to insert + + diffRunRow *run = diffRunRowAlloc(0, + "reg", + workdir, + label, + data_group ? data_group : label, + dist_group, + reduction, + NULL, // dvodb + registered, + tess_id, + true, // bothways + true, // exposure + false, // magicked + IPP_DIFF_MODE_WARP_WARP, + note); // Run to insert if (!diffRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1606,4 +1837,303 @@ } +static bool definestackstackMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *stack1Where = psMetadataAlloc(); + psMetadata *stack2Where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, stack1Where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-input_label", "stackRun.label","=="); + PXOPT_COPY_STR(config->args, stack2Where, "-template_label", "stackRun.label","=="); + PXOPT_COPY_F32(config->args, stack1Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); + PXOPT_COPY_F32(config->args, stack2Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(newTemplates,config->args,"-new-templates", false); + + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + if (!(label)) { + PXOPT_LOOKUP_STR(label,config->args, "-input_label",true,false); + } + + // Organize the config information into queries. + psString stack1Query = NULL; + psString stack2Query = NULL; + + if (psListLength(stack1Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(stack1Where, NULL); + psStringAppend(&stack1Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + stack1Query = psStringCopy("\n"); + } + psFree(stack1Where); + + if (psListLength(stack2Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(stack2Where, NULL); + psStringAppend(&stack2Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + stack2Query = psStringCopy("\n"); + } + psFree(stack2Where); + + // don't queue for stacks that have already been diffed unless requested + psString diffQuery = NULL; + if (! (reRun || newTemplates) ) { + psStringAppend(&diffQuery, "\n AND diffExp.diff_id IS NULL"); + } else { + diffQuery = psStringCopy("\n"); + } + + // find the distinct set of data_groups and filters + psString query = pxDataGet("difftool_definestackstack_part0.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psTrace("difftool",1,query,stack2Query,stack1Query,diffQuery,stack1Query); +/* exit(10); */ + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, stack2Query, stack1Query, diffQuery, stack1Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + query = pxDataGet("difftool_definestackstack_part1.sql"); + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRunTemp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs defined, to print + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Output row from query + bool mdok; // Status of MD lookup + + psString tess_id = psMetadataLookupStr(&mdok,row,"INPUT_tess_id"); + psString this_data_group = psMetadataLookupStr(&mdok,row,"INPUT_data_group"); + psString this_dist_group = psMetadataLookupStr(&mdok,row,"INPUT_dist_group"); + psString this_label = psMetadataLookupStr(&mdok,row,"INPUT_label"); + + psString this_stack1Query = psStringCopy(stack1Query); + + psString thisWhere = psDBGenerateWhereConditionSQL(row,NULL); + psStringSubstitute(&thisWhere,"stackRun.","INPUT_"); + psStringAppend(&this_stack1Query,"\n AND %s", thisWhere); + psFree(thisWhere); + + psTrace("difftool",1, query,stack2Query,this_stack1Query,diffQuery,this_stack1Query); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, stack2Query, this_stack1Query, diffQuery, this_stack1Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(this_stack1Query); + + psArray *diff_id_output = p_psDBFetchResult(config->dbh); + if (!diff_id_output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(diff_id_output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(diff_id_output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + // ok we've got one create the diffRun + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + label ? label : this_label, + data_group ? data_group : this_data_group, + dist_group ? dist_group : this_dist_group, + reduction, + NULL, // dvodb + registered, + tess_id, + false, // bothways + false, // exposure + 0, // magicked + IPP_DIFF_MODE_STACK_STACK, // diff_mode + note + ); + // Commit to database + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } +/* diffRunPrintObject(stdout,run,1); */ + run->diff_id = psDBLastInsertID(config->dbh); + for (long j = 0; j < diff_id_output->n; j++) { + psMetadata *input_row = diff_id_output->data[j]; // Output row from query + bool mdok; + psString in_skycell_id = psMetadataLookupStr(&mdok,input_row,"skycell_id"); + psS64 in_input_stack_id = psMetadataLookupS64(&mdok,input_row,"stack_id"); + psS64 in_template_stack_id = psMetadataLookupS64(&mdok,input_row,"max_stack_id"); + psString in_tess_id = psMetadataLookupStr(&mdok,input_row,"tess_id"); + psTrace("difftool",1,"%s %" PRId64 " %" PRId64 " %s\n", + in_skycell_id, in_input_stack_id, in_template_stack_id, in_tess_id); + diffInputSkyfileRow *input = diffInputSkyfileRowAlloc( + run->diff_id, // ID + in_skycell_id, + PS_MAX_S64, // warp1_id -> NULL + in_input_stack_id, // stack1 + PS_MAX_S64, // warp2_id -> NULL + in_template_stack_id, // stack2 + in_tess_id, + 0 // diff_skyfile_id + ); + // Commit to database the input + if (!diffInputSkyfileInsertObject(config->dbh, input)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(input); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + +/* diffInputSkyfilePrintObject(stdout,input,1); */ + psFree(input); + } + + if (!setdiffRunState(config, run->diff_id, "new", false)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, run->diff_id); + psFree(stack1Query); + psFree(stack2Query); + psFree(diffQuery); + psFree(run); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psArrayAdd(list, list->n, run); + psFree(run); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(query); + psFree(stack1Query); + psFree(stack2Query); + psFree(diffQuery); + psFree(output); + psFree(list); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + return(true); +} static bool pendingcleanuprunMode(pxConfig *config) @@ -1619,12 +2149,20 @@ psString query = pxDataGet("difftool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // pxDataGet/psStringSubstitute workaround + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereClause, " AND "); + psStringSubstitute(&query,whereClause,"@INNERCONSTRAINT@"); psFree(whereClause); + } + else { + psStringSubstitute(&query,NULL,"@INNERCONSTRAINT@"); } psFree(where); @@ -1636,5 +2174,7 @@ psFree(limitString); } - + // fprintf(stderr,"%s",query); + + //fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1684,5 +2224,5 @@ psString query = pxDataGet("difftool_pendingcleanupskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1744,5 +2284,5 @@ psString query = pxDataGet("difftool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1799,4 +2339,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) { @@ -1810,49 +2351,85 @@ } -static bool diffRunComplete(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // look for completed diffRuns - psString query = pxDataGet("difftool_completed_runs.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("difftool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); - psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); - - // set diffRun.state to 'stop' - if (!setdiffRunState(config, diff_id, "full", magicked)) { - psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, - diff_id); - psFree(output); - return false; - } - } - - return true; -} +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) { + PS_ASSERT_PTR_NON_NULL(config, false); + + // diff_id, skycell_id are required + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + + psS64 magicked = 0; + if (!strcmp(data_state, "full")) { + PXOPT_LOOKUP_S64(set_magicked, config->args, "-magicked", 0, false); + magicked = set_magicked; + } + + psString query = pxDataGet("difftool_change_skyfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + psString set_magicked_skyfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (magicked != 0 && !strcmp(data_state, "full")) { + psStringAppend(&set_magicked_skyfile, "\n , diffSkyfile.magicked = %" PRId64, magicked); + psStringAppend(&set_magicked_run, "\n , diffRun.magicked = %" PRId64, magicked); + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + psStringAppend(&set_magicked_skyfile, "\n, diffSkyfile.magicked = IF(diffSkyfile.magicked = 0, 0, -1)"); + psStringAppend(&set_magicked_run, "\n, diffRun.magicked = IF(diffRun.magicked = 0, 0, -1)"); + } + + // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, diff_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_skyfile); + return(false); + } + psFree(set_magicked_skyfile); + psFree(query); + + query = pxDataGet("difftool_change_run_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, diff_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_run); + return(false); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_run); + return(false); + } + psFree(set_magicked_run); + + return(true); +} + +static bool tocleanedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "cleaned","goto_cleaned"); +} +static bool topurgedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); +} +static bool tofullskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "full", "update"); +} + bool exportrunMode(pxConfig *config) @@ -1878,4 +2455,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); @@ -1890,5 +2471,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1969,7 +2550,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "diffRun"); psAssert (item, "entry not in input?"); @@ -2021,2 +2608,148 @@ return true; } + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "diffRun.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + if (!template) { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "LIKE"); + } else { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "LIKE"); + } + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "diffRun"); + pxspaceAddWhere(config, &where2, template ? "rawTemplate" : "rawInput"); + psString query = pxDataGet("difftool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (where2) { + psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + if (template) { + psStringAppend(&query, " ORDER BY rawTemplate.exp_id, diff_id DESC"); + } else { + psStringAppend(&query, " ORDER BY rawInput.exp_id, diff_id DESC"); + } + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("difftool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , diffRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/difftool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/difftool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/difftool.h (revision 27840) @@ -31,4 +31,5 @@ DIFFTOOL_MODE_TODIFFSKYFILE, DIFFTOOL_MODE_ADDDIFFSKYFILE, + DIFFTOOL_MODE_ADVANCE, DIFFTOOL_MODE_DIFFSKYFILE, DIFFTOOL_MODE_REVERTDIFFSKYFILE, @@ -36,4 +37,5 @@ DIFFTOOL_MODE_DEFINEWARPSTACK, DIFFTOOL_MODE_DEFINEWARPWARP, + DIFFTOOL_MODE_DEFINESTACKSTACK, DIFFTOOL_MODE_PENDINGCLEANUPRUN, DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, @@ -41,5 +43,11 @@ DIFFTOOL_MODE_UPDATEDIFFSKYFILE, DIFFTOOL_MODE_EXPORTRUN, - DIFFTOOL_MODE_IMPORTRUN + DIFFTOOL_MODE_IMPORTRUN, + DIFFTOOL_MODE_TOCLEANEDSKYFILE, + DIFFTOOL_MODE_TOPURGEDSKYFILE, + DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, + DIFFTOOL_MODE_TOFULLSKYFILE, + DIFFTOOL_MODE_LISTRUN, + DIFFTOOL_MODE_SETSKYFILETOUPDATE, } difftoolMode; Index: branches/simtest_nebulous_branches/ippTools/src/difftoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/difftoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/difftoolConfig.c (revision 27840) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -47,17 +47,29 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways", 0, "do the subtraction both ways?", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure", 0, "subtraction for entire exposure?", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways", 0, "do the subtraction both ways?", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure", 0, "subtraction for entire exposure?", false); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddS16(definerunArgs, PS_LIST_TAIL, "-set_diff_mode", 0, "specify type of difference (WW=1,WS=2,SW=3,SS=4)", 0); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define by label instead of diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "define by data_group instead of diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_diff_mode", 0, "specify type of difference (WW=1,WS=2,SW=3,SS=4)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -addinputskyfile @@ -113,23 +125,66 @@ psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); - psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", false); + psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); + + // -advance + psMetadata *advanceArgs = psMetadataAlloc(); + psMetadataAddS64(advanceArgs, PS_LIST_TAIL, "-diff_id", 0, "select by diff ID", 0); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "select by label", NULL); + psMetadataAddS32(advanceArgs, PS_LIST_TAIL, "-limit", 0, "limit number of results", 0); // -diffskyfile psMetadata *diffskyfileArgs = psMetadataAlloc(); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_skyfile_id", 0, "search by diff_skyfile_id ID", 0); - psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "define exposure name", NULL); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id", 0, "define warp_id", NULL); - psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_skyfile_id", 0, "search by diff_skyfile_id ID", 0); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", NULL); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-template", 0, "apply exposure args to template of bothways diff", false); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL); + psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxmagicAddArguments(diffskyfileArgs); + pxspaceAddArguments(diffskyfileArgs); + + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", NULL); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-template", 0, "apply exposure args to template of bothways diff", false); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL); + pxmagicAddArguments(listrunArgs); + pxspaceAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -revertdiffskyfile psMetadata *revertdiffskyfileArgs = psMetadataAlloc(); psMetadataAddS64(revertdiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); - psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); psMetadataAddS16(revertdiffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddBool(revertdiffskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); @@ -156,14 +211,19 @@ psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-comment", 0, "search by comment (LIKE)", NULL); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); - psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if no stacks available for some skycells", false); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false); @@ -182,10 +242,12 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-template_label", 0, "search by warp label for template", NULL); psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); - psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false); @@ -220,7 +282,30 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + // -definestackstack + psMetadata *definestackstackArgs = psMetadataAlloc(); + // stack id and exp id searches seem less useful here + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_label", 0, "search by stack label for input", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-template_label", 0, "search by stack label for template", NULL); + psMetadataAddF32(definestackstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definestackstackArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if no stacks available for some skycells", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definestackstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false); + + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); - psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list runs for cleanup with specified label", NULL); psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -241,5 +326,35 @@ psMetadata *updatediffskyfileArgs = psMetadataAlloc(); psMetadataAddS64(updatediffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); - psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + + // -tocleanedskyfile + psMetadata *tocleanedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(tocleanedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -topurgedskyfile + psMetadata *topurgedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -toscrubbedskyfile + psMetadata *toscrubbedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -tofullskyfile + psMetadata *tofullskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", 0); + + // -setskyfiletoupdate + psMetadata *setskyfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setskyfiletoupdateArgs, PS_LIST_TAIL, "-diff_id", 0, "search by difftool ID (required)", 0); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "new value for diffRun.label", NULL); + + // -exportrun @@ -267,15 +382,25 @@ PXOPT_ADD_MODE("-todiffskyfile", "", DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileArgs); PXOPT_ADD_MODE("-adddiffskyfile", "", DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileArgs); + PXOPT_ADD_MODE("-advance", "", DIFFTOOL_MODE_ADVANCE, advanceArgs); PXOPT_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); + PXOPT_ADD_MODE("-listrun", "", DIFFTOOL_MODE_LISTRUN, listrunArgs); PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); PXOPT_ADD_MODE("-definewarpstack", "", DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackArgs); PXOPT_ADD_MODE("-definewarpwarp", "", DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpArgs); + PXOPT_ADD_MODE("-definestackstack", "", DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackArgs); PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", DIFFTOOL_MODE_DONECLEANUP, donecleanupArgs); PXOPT_ADD_MODE("-updatediffskyfile", "update fault code for a diffskyfile", DIFFTOOL_MODE_UPDATEDIFFSKYFILE, updatediffskyfileArgs); + PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", DIFFTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", DIFFTOOL_MODE_IMPORTRUN, importrunArgs); + + PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); + PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); + PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/simtest_nebulous_branches/ippTools/src/disttool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/disttool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/disttool.c (revision 27840) @@ -2,5 +2,5 @@ * disttool.c * - * Copyright (C) 2008 + * Copyright (C) 2008-2009 * * This program is free software; you can redistribute it and/or modify it @@ -32,9 +32,9 @@ static bool definebyqueryMode(pxConfig *config); -static bool definerunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool revertrunMode(pxConfig *config); static bool pendingcomponentMode(pxConfig *config); static bool addprocessedcomponentMode(pxConfig *config); +static bool revertcomponentMode(pxConfig *config); static bool processedcomponentMode(pxConfig *config); static bool toadvanceMode(pxConfig *config); @@ -42,15 +42,15 @@ static bool addfilesetMode(pxConfig *config); static bool revertfilesetMode(pxConfig *config); +static bool updatefilesetMode(pxConfig *config); static bool queuercrunMode(pxConfig *config); static bool updatercrunMode(pxConfig *config); static bool revertrcrunMode(pxConfig *config); static bool pendingdestMode(pxConfig *config); +static bool pendingcleanupMode(pxConfig *config); +static bool listfilesetsMode(pxConfig *config); static bool definetargetMode(pxConfig *config); static bool updatetargetMode(pxConfig *config); -static bool listtargetMode(pxConfig *config); - -static bool definedsproductMode(pxConfig *config); -static bool updatedsproductMode(pxConfig *config); +static bool listtargetsMode(pxConfig *config); static bool definedestinationMode(pxConfig *config); @@ -59,4 +59,5 @@ static bool defineinterestMode(pxConfig *config); static bool updateinterestMode(pxConfig *config); +static bool listinterestsMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -80,5 +81,4 @@ switch (config->mode) { MODECASE(DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(DISTTOOL_MODE_DEFINERUN, definerunMode); MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode); @@ -86,8 +86,12 @@ MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode); MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode); + MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode); MODECASE(DISTTOOL_MODE_TOADVANCE, toadvanceMode); MODECASE(DISTTOOL_MODE_PENDINGFILESET, pendingfilesetMode); + MODECASE(DISTTOOL_MODE_PENDINGCLEANUP, pendingcleanupMode); MODECASE(DISTTOOL_MODE_ADDFILESET, addfilesetMode); MODECASE(DISTTOOL_MODE_REVERTFILESET, revertfilesetMode); + MODECASE(DISTTOOL_MODE_LISTFILESETS, listfilesetsMode); + MODECASE(DISTTOOL_MODE_UPDATEFILESET, updatefilesetMode); MODECASE(DISTTOOL_MODE_QUEUERCRUN, queuercrunMode); MODECASE(DISTTOOL_MODE_UPDATERCRUN, updatercrunMode); @@ -96,11 +100,10 @@ MODECASE(DISTTOOL_MODE_DEFINETARGET, definetargetMode); MODECASE(DISTTOOL_MODE_UPDATETARGET, updatetargetMode); - MODECASE(DISTTOOL_MODE_LISTTARGET, listtargetMode); - MODECASE(DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductMode); - MODECASE(DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductMode); + MODECASE(DISTTOOL_MODE_LISTTARGETS, listtargetsMode); MODECASE(DISTTOOL_MODE_DEFINEDESTINATION, definedestinationMode); MODECASE(DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationMode); MODECASE(DISTTOOL_MODE_DEFINEINTEREST, defineinterestMode); MODECASE(DISTTOOL_MODE_UPDATEINTEREST, updateinterestMode); + MODECASE(DISTTOOL_MODE_LISTINTERESTS, listinterestsMode); default: psAbort("invalid option (this should not happen)"); @@ -124,45 +127,4 @@ } -static bool definerunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); - PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); - PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false); - PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", true, false); - - // optional - PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); - PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); - PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); - - // TODO: check that stage has an expected value - // XXX: all of the following concerns will be managed properly by definebyquery - - // TODO: should we check that stage_id actually exists for stage - // in magicdstool we queue off of a magic_id so the stage_id, exp_id, and cam_id get looked up - // when the run is queued - - if (!distRunInsert(config->dbh, - 0, // dist_id - target_id, - stage, - stage_id, - set_label, - outroot, - clean, - no_magic, - "new", - NULL, // time_stamp - 0 // fault - )) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} static bool definebyqueryMode(pxConfig *config) @@ -175,11 +137,25 @@ // optional - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); + PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false); PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(set_note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + if (use_alternate) { + if (strcmp(stage, "raw")) { + psError(PXTOOLS_ERR_SYS, true, "alternate inputs only supported for raw stage"); + return false; + } + if (no_magic) { + psError(PXTOOLS_ERR_SYS, true, "no_magic forbidden with alternate inputs"); + return false; + } + } // select arguments @@ -191,4 +167,5 @@ PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); psString query = NULL; @@ -196,16 +173,24 @@ psString runJoinStr = NULL; if (!strcmp(stage, "raw")) { - magicRunType = "rawExp"; - runJoinStr = "rawExp.exp_id"; + if (! use_alternate ) { + magicRunType = "rawExp"; + runJoinStr = "rawExp.exp_id"; + } else { + magicRunType = "camRun"; + runJoinStr = "camRun.exp_id"; + } query = pxDataGet("disttool_definebyquery_raw.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; } - // for raw stage we select by chipRun.label because raw labels typically aren't set + // for raw stage we select by camRun.label and dist_group because rawExp doesn't have those columns if (label) { - psStringAppend(&query, " AND (chipRun.label = '%s')", label); + psStringAppend(&query, " AND (magicDSRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (camRun.dist_group = '%s')", dist_group); } } else if (!strcmp(stage, "chip")) { @@ -214,5 +199,5 @@ query = pxDataGet("disttool_definebyquery_chip.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -222,9 +207,13 @@ psStringAppend(&query, " AND (chipRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (chipRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "camera")) { - magicRunType = "chipRun"; // This is used below to set the magicked business + magicRunType = "camRun"; // This is used below to set the magicked business + runJoinStr = "camRun.cam_id"; query = pxDataGet("disttool_definebyquery_camera.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -233,4 +222,7 @@ if (label) { psStringAppend(&query, " AND (camRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (camRun.dist_group = '%s')", dist_group); } } else if (!strcmp(stage, "fake")) { @@ -238,5 +230,5 @@ query = pxDataGet("disttool_definebyquery_fake.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -245,4 +237,7 @@ if (label) { psStringAppend(&query, " AND (fakeRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (fakeRun.dist_group = '%s')", dist_group); } // fake stage doesn't require magic @@ -253,5 +248,5 @@ query = pxDataGet("disttool_definebyquery_warp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -260,4 +255,7 @@ if (label) { psStringAppend(&query, " AND (warpRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (warpRun.dist_group = '%s')", dist_group); } @@ -267,5 +265,5 @@ query = pxDataGet("disttool_definebyquery_diff.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -274,4 +272,7 @@ if (label) { psStringAppend(&query, " AND (diffRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (diffRun.dist_group = '%s')", dist_group); } @@ -280,5 +281,5 @@ query = pxDataGet("disttool_definebyquery_stack.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -288,10 +289,38 @@ psStringAppend(&query, " AND (stackRun.label = '%s')", label); } - // stack stage doesn't require magic (perhaps let the script do this? + if (dist_group) { + psStringAppend(&query, " AND (stackRun.dist_group = '%s')", dist_group); + } + // stack stage doesn't require magic no_magic = true; + } else if (!strcmp(stage, "SSdiff")) { + magicRunType = "diffRun"; + runJoinStr = "diffRun.diff_id"; + query = pxDataGet("disttool_definebyquery_SSdiff.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (diffRun.label = '%s') ", label); + } + if (dist_group) { + psStringAppend(&query, " AND (diffRun.dist_group = '%s') ", dist_group); + } + + no_magic = true; } else { psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); psFree(where); return false; + } + + if (!strcmp(stage, "raw")) { + if (!no_magic) { + psStringAppend(&query, " AND (magicDSRun.re_place = %d)", !use_alternate); + psStringAppend(&query, " AND (camRun.state = 'full')"); + } } @@ -306,26 +335,5 @@ if (!no_magic) { - psStringAppend(&query, " AND (distTarget.clean OR %s.magicked)", magicRunType); - - // is selecting by magic_ds_id really interesting? - if (magic_ds_id) { - if (strcmp(stage, "camera")) { - // stage other than camera - if (!runJoinStr) { - psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); - psFree(query); - return false; - } - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" - " AND magicDSRun.stage_id = %s", runJoinStr); - } else { - // camera masks are magicked when the chipRun is magicked - // XXX: This is confusing. Is it dangerous? - // Maybe I should add a magicked bit to camRun. Note this isn't - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = 'chip'" - " AND magicDSRun.stage_id = chipRun.chip_id"); - } - psStringAppend(&query, " AND (magicDSRun.state = 'full' AND magicDSRun.re_place AND (magic_ds_id = %" PRId64 "))", magic_ds_id); - } + psStringAppend(&query, " AND (%s.magicked > 0)", magicRunType); } @@ -335,9 +343,9 @@ psFree(limitString); } - + psTrace("disttool",2,query,joinHook ? joinHook : ""); if (!p_psDBRunQueryF(config->dbh, query, joinHook ? joinHook : "")) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; } psFree(query); @@ -356,5 +364,5 @@ - if (dry_run) { + if (pretend) { if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", true)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -377,10 +385,12 @@ psString run_tag = psMetadataLookupStr(NULL, md, "run_tag"); psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); + psS64 magic_ds_id = psMetadataLookupS64(NULL, md, "magicked"); psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); - psString target_label = psMetadataLookupStr(NULL, md, "label"); + psString stage_label = psMetadataLookupStr(NULL, md, "label"); + psString stage_data_group = psMetadataLookupStr(NULL, md, "data_group"); bool clean = psMetadataLookupBool(NULL, md, "clean"); psString outroot = NULL; - psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage); + psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage); psString new_label; @@ -388,6 +398,13 @@ new_label = set_label; } else { - new_label = target_label; - } + new_label = stage_label; + } + psString new_data_group; + if (set_data_group != NULL) { + new_data_group = set_data_group; + } else { + new_data_group = stage_data_group; + } + distRunRow *row = distRunRowAlloc( 0, // dist_id @@ -395,11 +412,16 @@ stage, stage_id, + magic_ds_id, new_label, outroot, + NULL, // outdir clean, no_magic, + use_alternate, "new", - NULL, // time_stamp - 0 // fault + NULL, // time_stamp + 0, // fault + new_data_group, + set_note // note does not propagate ); @@ -444,10 +466,13 @@ PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; - PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "distRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "distRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "distTarget.dist_group", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -455,25 +480,40 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - - if ((!state) && (!label) && (!fault)) { - psError(PXTOOLS_ERR_DATA, false, "parameters (-fault or -set_state or -set_label) are required"); - psFree(where); - return false; - } - - psString query = psStringCopy("UPDATE distRun SET time_stamp = UTC_TIMESTAMP()"); - + PXOPT_LOOKUP_STR(set_note, config->args, "-set_note", false, false); + + if ((!state) && (!label) && (!fault) &&(!data_group)) { + psError(PXTOOLS_ERR_CONFIG, false, "parameters (-fault or -set_state or -set_label -set_data_group) are required"); + psFree(where); + return false; + } + + psString query = psStringCopy("UPDATE distRun join distTarget using(target_id, stage) SET distRun.time_stamp = UTC_TIMESTAMP()"); + + if (outdir) { + psStringAppend(&query, " , distRun.outdir = '%s'", outdir); + } if (state) { - psStringAppend(&query, " , state = '%s'", state); + psStringAppend(&query, " , distRun.state = '%s'", state); } if (label) { - psStringAppend(&query, " , label = '%s'", label); + psStringAppend(&query, " , distRun.label = '%s'", label); } if (fault) { - psStringAppend(&query, " , fault = %d", fault); - } + psStringAppend(&query, " , distRun.fault = %d", fault); + } + + if (data_group) { + psStringAppend(&query, " , distRun.data_group = '%s'", data_group); + } + + if (set_note) { + psStringAppend(&query, " , distRun.note = '%s'", set_note); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -500,8 +540,5 @@ PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - // we need to disambiguate fault so make a copy of the where list before adding fault - psMetadata *whereComponent = psMetadataCopy(NULL, where); PXOPT_COPY_S16(config->args, where, "-fault", "distRun.fault", "=="); - PXOPT_COPY_S16(config->args, whereComponent, "-fault", "distComponent.fault", "=="); // It might be useful to be able to query by the parameters of the underlying runs @@ -509,88 +546,81 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Update state to 'new' - int numUpdated; // Number updated - { - psString query = pxDataGet("disttool_revertrun_update.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("disttool_revertrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numUpdated = psDBAffectedRows(config->dbh); - } + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + int numUpdated = psDBAffectedRows(config->dbh); psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated); - // Delete product - int numDeleted; // Number deleted - { - psString query = pxDataGet("disttool_revertrun_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(whereComponent->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(whereComponent, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } + return true; +} + +static bool revertcomponentMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; + PXOPT_COPY_STR(config->args, where, "-component", "component", "==");; + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + pxAddLabelSearchArgs(config, where, "-label", "distRun.label", "=="); + + PXOPT_COPY_S16(config->args, where, "-fault", "distComponent.fault", "=="); + + // It might be useful to be able to query by the parameters of the underlying runs + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("disttool_revertcomponent.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("disttool", PS_LOG_INFO, "Deleted %d distComponents", numDeleted); - psFree(where); - psFree(whereComponent); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - return true; } @@ -600,16 +630,18 @@ PS_ASSERT_PTR_NON_NULL(config, false); - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("disttool_pendingcomponent.sql"); + psString queryFile = NULL; + psStringAppend(&queryFile, "disttool_pending_%s.sql", stage); + psString query = pxDataGet(queryFile); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement from %s", queryFile); return false; } @@ -617,5 +649,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -629,28 +661,5 @@ } - // the query has where hooks for each stage. - // right now we aren't using them. - // XXX: I think that I want to change the query from a union of selects on the various - // stages to separate queries. As it is pending data at the later stages of the pipline - // will get blocked by pending earlier stages - psString raw_where = ""; - psString raw_clean_where = ""; - psString chip_where = ""; - psString camera_where = ""; - psString fake_where = ""; - psString warp_where = ""; - psString diff_where = ""; - psString stack_where = ""; - - if (!p_psDBRunQueryF(config->dbh, - query, - raw_where, - raw_clean_where, - chip_where, - camera_where, - fake_where, - warp_where, - diff_where, - stack_where)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -708,7 +717,8 @@ PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", require_fileinfo, false); PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", require_fileinfo, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", require_fileinfo, false); PXOPT_LOOKUP_STR(name, config->args, "-name", require_fileinfo, false); - if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", name, fault)) { + if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", outdir, name, fault)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -732,5 +742,5 @@ psString query = pxDataGet("disttool_toadvance.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -803,5 +813,5 @@ psString query = pxDataGet("disttool_processedcomponent.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -868,4 +878,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "distRun.stage", "=="); pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); @@ -876,5 +887,5 @@ psString query = pxDataGet("disttool_pendingfileset.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -939,5 +950,5 @@ // required values PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false); - PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -946,8 +957,8 @@ PXOPT_LOOKUP_STR(name, config->args, "-name", fault == 0, false); - if (!rcDSFilesetInsert(config->dbh, + if (!rcDSFilesetInsert(config->dbh, 0, // fs_id dist_id, - prod_id, + dest_id, name, "full", @@ -959,4 +970,5 @@ return true; } + static bool revertfilesetMode(pxConfig *config) { @@ -964,5 +976,5 @@ PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); PXOPT_COPY_S64(config->args, where, "-dist_id", "rcDSFileset.dist_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); @@ -975,5 +987,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -981,5 +993,5 @@ psString query = pxDataGet("disttool_revertfileset.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -1023,5 +1035,5 @@ psString query = pxDataGet("disttool_pendingdest.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1092,5 +1104,4 @@ PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); PXOPT_COPY_S64(config->args, where, "-target_id","target_id", "=="); PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); @@ -1103,5 +1114,5 @@ psString query = pxDataGet("disttool_queuercrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1149,5 +1160,5 @@ // We either need rc_id or (dest_id and fs_name) to identifiy the rcRun if ((!rc_id) && !(dest_id && fs_name)) { - psError(PXTOOLS_ERR_DATA, true, "either -rc_id or (-fs_name and -dest_id) are required"); + psError(PXTOOLS_ERR_CONFIG, true, "either -rc_id or (-fs_name and -dest_id) are required"); return false; } @@ -1161,5 +1172,5 @@ // this can't happen because we checked above psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -1171,5 +1182,5 @@ if (!state && (fault < 0)) { - psError(PXTOOLS_ERR_DATA, false, "parameters (-fault or -set_state) are required"); + psError(PXTOOLS_ERR_CONFIG, false, "parameters (-fault or -set_state) are required"); psFree(where); return false; @@ -1178,5 +1189,5 @@ psString query = pxDataGet("disttool_updatercrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1235,5 +1246,5 @@ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -1241,5 +1252,5 @@ psString query = pxDataGet("disttool_revertrcrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1276,5 +1287,5 @@ // required - PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", true, false); PXOPT_LOOKUP_STR(filter, config->args, "-filter", true, false); PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); @@ -1287,5 +1298,5 @@ distTargetRow *row = distTargetRowAlloc( 0, // target_id - label, + dist_group, filter, stage, @@ -1294,5 +1305,5 @@ comment ); - + if (!row) { psError(PS_ERR_UNKNOWN, false, "failed to allocate distTarget object"); @@ -1324,5 +1335,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); @@ -1354,5 +1365,5 @@ } -static bool listtargetMode(pxConfig *config) +static bool listtargetsMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -1360,5 +1371,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); @@ -1433,27 +1444,37 @@ } -static bool definedsproductMode(pxConfig *config) +static bool definedestinationMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); - PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); - PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); + PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); + + // optional + PXOPT_LOOKUP_STR(status_uri, config->args, "-status_uri", false, false); + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + PXOPT_LOOKUP_STR(last_fileset, config->args, "-last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); // XXX: should we insure that these names do not contatin any whitespace? - rcDSProductRow *row = rcDSProductRowAlloc( - 0, // prod_id + rcDestinationRow *row = rcDestinationRowAlloc( + 0, // dest_id name, + status_uri, + comment, + last_fileset, dbname, - dbhost + dbhost, + state ? state : "enabled" ); - + if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDSProduct object"); - return false; - } - if (!rcDSProductInsertObject(config->dbh, row)) { + psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDestination object"); + return false; + } + if (!rcDestinationInsertObject(config->dbh, row)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(row); @@ -1462,7 +1483,7 @@ // get the assigned target_id - row->prod_id = psDBLastInsertID(config->dbh); - - if (!rcDSProductPrintObject(stdout, row, true)) { + row->dest_id = psDBLastInsertID(config->dbh); + + if (!rcDestinationPrintObject(stdout, row, true)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); psFree(row); @@ -1474,28 +1495,37 @@ return true; } -static bool updatedsproductMode(pxConfig *config) + +static bool updatedestinationMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); - - PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", false, false); - PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", false, false); - - if (!(dbname || dbhost)) { - psError(PS_ERR_UNKNOWN, true, "one or more of dbname or dbhost is required"); - psFree(where); - return false; - } - psString query = psStringCopy("UPDATE rcDSProduct SET"); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); +#ifdef ALLOW_UPDATE_LAST_FILESET + PXOPT_LOOKUP_STR(last_fileset, config->args, "-set_last_fileset", false, false); + if (!(state || last_fileset)) { + psError(PS_ERR_UNKNOWN, true, "one or more of -set_state or -set_last_fileset is required"); +# else + if (!state) { +#endif + psFree(where); + return false; + } + psString query = psStringCopy("UPDATE rcDestination SET"); psString sep = ""; - if (dbname) { - psStringAppend(&query, " dbname = '%s'", dbname); + if (state) { + psStringAppend(&query, " state = '%s'", state); sep = ","; } - if (dbhost) { - psStringAppend(&query, " %s dbhost = '%s'", sep, dbhost); - } +#ifdef ALLOW_UPDATE_LAST_FILESET + // last_fileset normally gets set by updatercrunMode + // Allowing it to be set here might cause problems + // especially since we are allowing selection by dest_id + if (last_fileset) { + psStringAppend(&query, " %s last_fileset = '%s'", sep, last_fileset); + } +#endif if (psListLength(where->list)) { @@ -1521,87 +1551,142 @@ } -static bool definedestinationMode(pxConfig *config) +static bool defineinterestMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); - PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + // one of these is required + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", false, false); + PXOPT_LOOKUP_STR(dest_name, config->args, "-dest_name", false, false); + if (!dest_id && !dest_name) { + psError(PS_ERR_UNKNOWN, true, "either dest_id or dest_name is required"); + return false; + } + + // either target_id or stage and label are required + PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + if (!target_id) { + bool error = false; + if (!stage) { + psError(PS_ERR_UNKNOWN, true, "stage is required if target_id is not supplied"); + error = true; + } + if (!dist_group) { + psError(PS_ERR_UNKNOWN, !error, "dist_group is required if target_id is not supplied"); + error = true; + } + if (error) { + return false; + } + } // optional - PXOPT_LOOKUP_STR(status_uri, config->args, "-status_uri", false, false); - PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); - PXOPT_LOOKUP_STR(last_fileset, config->args, "-last_fileset", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - - // XXX: should we insure that these names do not contatin any whitespace? - - rcDestinationRow *row = rcDestinationRowAlloc( - 0, // dest_id - prod_id, - name, - status_uri, - comment, - last_fileset, - state ? state : "enabled" - ); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDestination object"); - return false; - } - if (!rcDestinationInsertObject(config->dbh, row)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(row); - return false; - } - - // get the assigned target_id - row->dest_id = psDBLastInsertID(config->dbh); - - if (!rcDestinationPrintObject(stdout, row, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(row); - return false; - } - - psFree(row); - - return true; -} - -static bool updatedestinationMode(pxConfig *config) + PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + if (state) { + if (strcmp(state, "enabled") && strcmp(state, "disabled")) { + psError(PS_ERR_PROGRAMMING, true, "state must be enabled or disabled"); + return false; + } + } else { + // default state + state = "enabled"; + } + + // now that we've done all of our argument checking, copy the values to where + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + // if stage is all don't add it to the query (match all stages) + if (stage && strcmp(stage, "all")) { + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + } + + psString query = pxDataGet("disttool_defineinterest.sql"); + + if (!psListLength(where->list)) { + // can't get here + psError(PS_ERR_PROGRAMMING, true, "search parameters are required"); + psFree(where); + psFree(query); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + if (clean) { + psStringAppend(&query, " AND (distTarget.clean)"); + } else { + psStringAppend(&query, " AND (!distTarget.clean)"); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + { + // psStringSubstitute fails unless the input is a psString which it determines by + // comparing the memory blocks free function to an expected value. + // pxDataGet uses psSlurp which leaves a different free function on the memory block. + // To work around this make a copy of the query before doing the substitution. + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; + } + // change the @STATE@ in the sql file to our state + if (!psStringSubstitute(&query, state, "@STATE@")) { + psError(PS_ERR_UNKNOWN, false, "failed to substitute state string"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + int numInserted = psDBAffectedRows(config->dbh); + printf("inserted %d rows into rcInterest\n", numInserted); + + return true; +} + +static bool updateinterestMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); + PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + if (full && clean) { + psError(PS_ERR_UNKNOWN, true, "-full and -clean makes no sense, chose one or the other"); + psFree(where); + return false; + } + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); -#ifdef ALLOW_UPDATE_LAST_FILESET - PXOPT_LOOKUP_STR(last_fileset, config->args, "-set_last_fileset", false, false); - if (!(state || last_fileset)) { - psError(PS_ERR_UNKNOWN, true, "one or more of -set_state or -set_last_fileset is required"); -# else if (!state) { -#endif - psFree(where); - return false; - } - psString query = psStringCopy("UPDATE rcDestination SET"); - psString sep = ""; - if (state) { - psStringAppend(&query, " state = '%s'", state); - sep = ","; - } -#ifdef ALLOW_UPDATE_LAST_FILESET - // last_fileset normally gets set by updatercrunMode - // Allowing it to be set here might cause problems - // especially since we are allowing selection by prod_id - if (last_fileset) { - psStringAppend(&query, " %s last_fileset = '%s'", sep, last_fileset); - } -#endif + psError(PS_ERR_UNKNOWN, true, "-set_state is required"); + psFree(where); + return false; + } + psString query = pxDataGet("disttool_updateinterest.sql"); if (psListLength(where->list)) { @@ -1617,5 +1702,11 @@ psFree(where); - if (!p_psDBRunQuery(config->dbh, query)) { + if (clean) { + psStringAppend(&query, " AND distTarget.clean"); + } else { + psStringAppend(&query, " AND NOT distTarget.clean"); + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1624,68 +1715,35 @@ psFree(query); - return true; -} - -static bool defineinterestMode(pxConfig *config) + psS64 numUpdated = psDBAffectedRows(config->dbh); + printf("updated %" PRId64 " interests\n", numUpdated); + + return true; +} +static bool listinterestsMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); - PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", true, false); - - // optional - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - - // XXX: should we insure that these names do not contatin any whitespace? - - rcInterestRow *row = rcInterestRowAlloc( - 0, // int_id - dest_id, - target_id, - state ? state : "enabled" - ); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcInterest object"); - return false; - } - if (!rcInterestInsertObject(config->dbh, row)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(row); - return false; - } - - // get the assigned target_id - row->int_id = psDBLastInsertID(config->dbh); - - if (!rcInterestPrintObject(stdout, row, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(row); - return false; - } - - psFree(row); - - return true; -} - -static bool updateinterestMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); - PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "name", "=="); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - if (!state) { - psError(PS_ERR_UNKNOWN, true, "-set_state is required"); - psFree(where); - return false; - } - psString query = NULL; - psStringAppend(&query, "UPDATE rcInterest SET state = '%s'", state); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "rcInterest.state", "=="); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (clean && full) { + psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full"); + return false; + } + + psString query = pxDataGet("disttool_listinterests.sql"); if (psListLength(where->list)) { @@ -1693,20 +1751,237 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } else { - psError(PS_ERR_UNKNOWN, true, "search parameters are required"); - psFree(where); - psFree(query); - return false; + if (clean) { + psStringAppend(&query, " AND (clean)"); + } else if (full) { + psStringAppend(&query, " AND (!clean)"); + } + } else if (clean) { + psStringAppend(&query, " WHERE clean"); + } else if (full) { + psStringAppend(&query, " WHERE !clean"); } psFree(where); + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } psFree(query); - return true; -} - + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "rcInterest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool pendingcleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + psString query = pxDataGet("disttool_pendingcleanup.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + } else if (!all) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters or -all are required"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "distToCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool listfilesetsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("disttool_listfilesets.sql"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "distFilesets", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool updatefilesetMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + // We don't use PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); here + // because we want -fault 0 to work + bool gotFault = false; + psS16 fault = psMetadataLookupS16(&gotFault, config->args, "-fault"); + + if ((!state) && (!gotFault)) { + psError(PXTOOLS_ERR_CONFIG, true, "parameters (-fault or -set_state) is required"); + psFree(where); + return false; + } + + psString query = psStringCopy("UPDATE rcDSFileset SET "); + + if (state) { + psStringAppend(&query, " state = '%s'", state); + } + + if (gotFault) { + psStringAppend(&query, "%s fault = %d", state ? ", " : "", fault); + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/disttool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/disttool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/disttool.h (revision 27840) @@ -26,14 +26,17 @@ DISTTOOL_MODE_NONE = 0x0, DISTTOOL_MODE_DEFINEBYQUERY, - DISTTOOL_MODE_DEFINERUN, DISTTOOL_MODE_UPDATERUN, DISTTOOL_MODE_REVERTRUN, DISTTOOL_MODE_PENDINGCOMPONENT, DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, + DISTTOOL_MODE_REVERTCOMPONENT, DISTTOOL_MODE_PROCESSEDCOMPONENT, DISTTOOL_MODE_TOADVANCE, + DISTTOOL_MODE_PENDINGCLEANUP, DISTTOOL_MODE_PENDINGFILESET, DISTTOOL_MODE_ADDFILESET, DISTTOOL_MODE_REVERTFILESET, + DISTTOOL_MODE_LISTFILESETS, + DISTTOOL_MODE_UPDATEFILESET, DISTTOOL_MODE_QUEUERCRUN, DISTTOOL_MODE_UPDATERCRUN, @@ -42,11 +45,10 @@ DISTTOOL_MODE_DEFINEDESTINATION, DISTTOOL_MODE_UPDATEDESTINATION, - DISTTOOL_MODE_DEFINEDSPRODUCT, - DISTTOOL_MODE_UPDATEDSPRODUCT, DISTTOOL_MODE_DEFINETARGET, DISTTOOL_MODE_UPDATETARGET, - DISTTOOL_MODE_LISTTARGET, + DISTTOOL_MODE_LISTTARGETS, DISTTOOL_MODE_DEFINEINTEREST, DISTTOOL_MODE_UPDATEINTEREST, + DISTTOOL_MODE_LISTINTERESTS, } disttoolMode; Index: branches/simtest_nebulous_branches/ippTools/src/disttoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/disttoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/disttoolConfig.c (revision 27840) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psError(psErrorCodeLast(), false, "Can't find site configuration!\n"); psFree(config); return NULL; @@ -49,27 +49,20 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); - - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be selected", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group for run", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "select by dist_group", NULL); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); // select args - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define chip_id", 0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by label", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "define exp_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by run label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dist_group", 0, "select by dist_group", NULL); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -definerun - psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for bundle (required)", NULL); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id (required)", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output destination (required)", NULL); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-clean", 0, "build clean distribution bundle", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); // -updaterun @@ -78,9 +71,13 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_outdir", 0, "new value for outdir", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "value for stage", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "value for state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "limit updates to label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "limit updates to data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "limit updates to data_group", NULL); psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); - // -revertrun psMetadata *revertrunArgs = psMetadataAlloc(); @@ -95,9 +92,18 @@ // -pendingcomponent psMetadata *pendingcomponentArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage (required)", NULL); psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingcleanup + psMetadata *pendingcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(pendingcleanupArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); + psMetadataAddStr(pendingcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); + psMetadataAddU64(pendingcleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(pendingcleanupArgs, PS_LIST_TAIL, "-all", 0, "list all runs pending cleanup", NULL); @@ -106,8 +112,20 @@ psMetadataAddS64(addprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL); + psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory", NULL); psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0); psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-md5sum", 0, "define stage for bundle", NULL); psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + + // -revertcomponent + psMetadata *revertcomponentArgs = psMetadataAlloc(); + psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddS16(revertcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddBool(revertcomponentArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); // -processedcomponent @@ -128,4 +146,5 @@ psMetadataAddS64(pendingfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); + psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "limit results to dist_group", NULL); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -135,7 +154,14 @@ psMetadata *addfilesetArgs = psMetadataAlloc(); psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); + psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); psMetadataAddS32(addfilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + + // -updatefileset + psMetadata *updatefilesetArgs = psMetadataAlloc(); + psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); +// psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-set_state",0, "new value for state", NULL); + psMetadataAddS32(updatefilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); // -pendingdest @@ -150,5 +176,5 @@ psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define dist_id", 0); + psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dist_id", 0); psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0); @@ -158,9 +184,25 @@ psMetadataAddBool(revertfilesetArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); + // -listfileset + psMetadata *listfilesetsArgs = psMetadataAlloc(); + psMetadataAddS64(listfilesetsArgs, PS_LIST_TAIL, "-dist_id", 0, "list filesets with dist_id", 0); + psMetadataAddS64(listfilesetsArgs, PS_LIST_TAIL, "-target_id", 0, "list filesets with target_id", 0); + psMetadataAddS64(listfilesetsArgs, PS_LIST_TAIL, "-dest_id", 0, "list filesets with dest_id", 0); + psMetadataAddS64(listfilesetsArgs, PS_LIST_TAIL, "-int_id", 0, "list filesets with int_id", 0); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-dest_name", 0, "list filesets for destinationn name)", NULL); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-dist_group", 0, "list filesets for dist_group (LIKE comparison)", NULL); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-filter", 0, "list filesets by filter (LIKE comparison)", NULL); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-stage", 0, "list filesets for stage", NULL); + psMetadataAddBool(listfilesetsArgs, PS_LIST_TAIL,"-clean", 0, "list clean filesets", false); + psMetadataAddBool(listfilesetsArgs, PS_LIST_TAIL,"-full", 0, "list full filesets", false); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-state", 0, "list filesets in state", NULL); + psMetadataAddStr(listfilesetsArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); + psMetadataAddU64(listfilesetsArgs, PS_LIST_TAIL, "-limit", 0, "limit number of filesets listed to N", 0); + psMetadataAddBool(listfilesetsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -queuercrun psMetadata *queuercrunArgs = psMetadataAlloc(); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); - psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); @@ -189,22 +231,9 @@ psMetadataAddBool(revertrcrunArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); - // -definedsproduct - psMetadata *definedsproductArgs = psMetadataAlloc(); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-name", 0, "define product name", NULL); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); - - // -updatedsproduct - // does this mode make sense? - psMetadata *updatedsproductArgs = psMetadataAlloc(); - psMetadataAddS64(updatedsproductArgs, PS_LIST_TAIL, "-prod_id", 0, "select by prod_id", 0); - // can't select by name because it isn't necssarily unique - psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); - psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); - // -definedestination psMetadata *definedestinationArgs = psMetadataAlloc(); - psMetadataAddS64(definedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id (required)", 0); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host (required)", NULL); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); @@ -215,15 +244,12 @@ psMetadata *updatedestinationArgs = psMetadataAlloc(); psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); - psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); -// last_fileset gets updated by -updatercrun -// psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_last_fileset",0, "define last_fileset", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); // -definetarget psMetadata *definetargetArgs = psMetadataAlloc(); - psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-label", 0, "define label (required)", NULL); + psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group (required)", NULL); psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter (required)", NULL); psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); @@ -236,25 +262,32 @@ psMetadataAddS64(updatetargetArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); - psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); - psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group", NULL); psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); - // -listtarget - psMetadata *listtargetArgs = psMetadataAlloc(); - psMetadataAddS64(listtargetArgs, PS_LIST_TAIL, "-target_id", 0, "list target with target_id", 0); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-label", 0, "list targets for label", NULL); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-stage", 0, "list targets for stage", NULL); - psMetadataAddBool(listtargetArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); - psMetadataAddBool(listtargetArgs, PS_LIST_TAIL,"-full", 0, "list full targets", false); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-state", 0, "list targets in state", NULL); - psMetadataAddU64(listtargetArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); - psMetadataAddBool(listtargetArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + + // -listtargets + psMetadata *listtargetsArgs = psMetadataAlloc(); + psMetadataAddS64(listtargetsArgs, PS_LIST_TAIL, "-target_id", 0, "list target with target_id", 0); + psMetadataAddStr(listtargetsArgs, PS_LIST_TAIL, "-dist_group", 0, "list targets for dist_group", NULL); + psMetadataAddStr(listtargetsArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); + psMetadataAddStr(listtargetsArgs, PS_LIST_TAIL, "-stage", 0, "list targets for stage", NULL); + psMetadataAddBool(listtargetsArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); + psMetadataAddBool(listtargetsArgs, PS_LIST_TAIL,"-full", 0, "list full targets", false); + psMetadataAddStr(listtargetsArgs, PS_LIST_TAIL, "-state", 0, "list targets in state", NULL); + psMetadataAddU64(listtargetsArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); + psMetadataAddBool(listtargetsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -defineinterest psMetadata *defineinterestArgs = psMetadataAlloc(); - psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id (required)", 0); - psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-dest_name", 0, "define destination name (LIKE comparison)", NULL); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); + psMetadataAddBool(defineinterestArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddU64(defineinterestArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); // -updateinterest @@ -264,4 +297,25 @@ psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-dest_name", 0, "define destination name", NULL); + psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-dist_group", 0, "define distribution group", NULL); + psMetadataAddBool(updateinterestArgs, PS_LIST_TAIL,"-clean", 0, "update clean interests", false); + psMetadataAddBool(updateinterestArgs, PS_LIST_TAIL,"-full", 0, "update full interests", false); + + // -listinterests + psMetadata *listinterestsArgs = psMetadataAlloc(); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-target_id", 0, "list interests with target_id", 0); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-dest_id", 0, "list interests with dest_id", 0); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-int_id", 0, "list interests with int_id", 0); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dest_name", 0, "list interests for destinationn name)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dist_group", 0, "list interests for dist_group (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-filter", 0, "list interests by filter (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-stage", 0, "list interests for stage", NULL); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-clean", 0, "list clean interests", false); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-full", 0, "list full interests", false); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-state", 0, "list interests in state", NULL); + psMetadataAddU64(listinterestsArgs, PS_LIST_TAIL, "-limit", 0, "limit number of interests listed to N", 0); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadata *argSets = psMetadataAlloc(); @@ -269,13 +323,16 @@ PXOPT_ADD_MODE("-definebyquery", "", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "", DISTTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", DISTTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-revertrun", "", DISTTOOL_MODE_REVERTRUN, revertrunArgs); - PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); - PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); + PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-revertcomponent", "", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); PXOPT_ADD_MODE("-toadvance", "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs); + PXOPT_ADD_MODE("-pendingcleanup", "", DISTTOOL_MODE_PENDINGCLEANUP, pendingcleanupArgs); PXOPT_ADD_MODE("-pendingfileset", "", DISTTOOL_MODE_PENDINGFILESET, pendingfilesetArgs); PXOPT_ADD_MODE("-addfileset", "", DISTTOOL_MODE_ADDFILESET, addfilesetArgs); + PXOPT_ADD_MODE("-updatefileset", "", DISTTOOL_MODE_UPDATEFILESET, updatefilesetArgs); + PXOPT_ADD_MODE("-listfilesets", "", DISTTOOL_MODE_LISTFILESETS, listfilesetsArgs); PXOPT_ADD_MODE("-revertfileset", "", DISTTOOL_MODE_REVERTFILESET, revertfilesetArgs); PXOPT_ADD_MODE("-queuercrun", "", DISTTOOL_MODE_QUEUERCRUN, queuercrunArgs); @@ -284,8 +341,4 @@ PXOPT_ADD_MODE("-pendingdest", "", DISTTOOL_MODE_PENDINGDEST, pendingdestArgs); - PXOPT_ADD_MODE("-definedsproduct", "", DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductArgs); - PXOPT_ADD_MODE("-updatedsproduct", "", DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductArgs); -// PXOPT_ADD_MODE("-listdsproduct", "", DISTTOOL_MODE_LISTDSPRODUCT, updatedsproductArgs); - PXOPT_ADD_MODE("-definedestination", "", DISTTOOL_MODE_DEFINEDESTINATION, definedestinationArgs); PXOPT_ADD_MODE("-updatedestination", "", DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationArgs); @@ -294,9 +347,9 @@ PXOPT_ADD_MODE("-definetarget", "", DISTTOOL_MODE_DEFINETARGET, definetargetArgs); PXOPT_ADD_MODE("-updatetarget", "", DISTTOOL_MODE_UPDATETARGET, updatetargetArgs); - PXOPT_ADD_MODE("-listtarget", "", DISTTOOL_MODE_LISTTARGET, listtargetArgs); + PXOPT_ADD_MODE("-listtargets", "", DISTTOOL_MODE_LISTTARGETS, listtargetsArgs); PXOPT_ADD_MODE("-defineinterest", "", DISTTOOL_MODE_DEFINEINTEREST, defineinterestArgs); PXOPT_ADD_MODE("-updateinterest", "", DISTTOOL_MODE_UPDATEINTEREST, updateinterestArgs); -// PXOPT_ADD_MODE("-listinterest", "", DISTTOOL_MODE_LISTINTEREST, listinterestArgs); + PXOPT_ADD_MODE("-listinterests", "", DISTTOOL_MODE_LISTINTERESTS, listinterestsArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/simtest_nebulous_branches/ippTools/src/dqstatstool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dqstatstool.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/dqstatstool.c (revision 27840) @@ -0,0 +1,799 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdqstats.h" +#include "dqstatstool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool createbundleMode(pxConfig *config); +//static bool cleanbundleMode(pxConfig *config); +static bool pendingbundleMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool revertrunMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = dqstatstoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(DQSTATS_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DQSTATS_MODE_PENDINGBUNDLE, pendingbundleMode); + MODECASE(DQSTATS_MODE_CREATEBUNDLE, createbundleMode); + // MODECASE(DQSTATS_MODE_CLEANBUNDLE, cleanbundleMode); + MODECASE(DQSTATS_MODE_UPDATERUN, updaterunMode); + MODECASE(DQSTATS_MODE_REVERTRUN, revertrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + //psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(EXIT_SUCCESS); + + FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + // psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exit_status); +} + +static bool definebyqueryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,NULL); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(force, config->args, "-force", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, 0); + + // properties of the raw exposures + psMetadata *where = psMetadataAlloc(); + + pxdqstatsGetSearchArgs(config, where); + + psString query = pxDataGet("dqstatstool_definebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return(false); + } + + if (label) { + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + } + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (force) { + psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1 OR invalid = 0) "); + } + else { + psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1) "); + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + psTrace("debug",3,">> %s",query); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + // printf("%s\n",query); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("dqstatstool", PS_LOG_INFO, "no rows found"); + psWarning("dqstatstool: no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "dqstatsContent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + // Create the run entry and grab the dqstats_id + dqstatsRunRow *run = dqstatsRunRowAlloc( + 0, // ID + "new", // state + registered, + set_label, + 0 // fault + ); + + if (!dqstatsRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + return(false); + } + + psS64 dqstats_id = psDBLastInsertID(config->dbh); + + psArray *list = psArrayAllocEmpty(16); // List of runs to print + psArrayAdd(list, list->n, run); + if (!dqstatsRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return(false); + } + psFree(list); + + // Create the content entry. + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + psS64 exp_id = psMetadataLookupS64(&status, row, "exp_id"); + psS64 chip_id= psMetadataLookupS64(&status, row, "chip_id"); + psS64 cam_id = psMetadataLookupS64(&status, row, "cam_id"); + psS64 warp_id= psMetadataLookupS64(&status, row, "warp_id"); + + if (!dqstatsContentInsert(config->dbh, + dqstats_id, + exp_id, + chip_id, + cam_id, + warp_id, + 0 // do not create automatically invalid contents + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + return(true); +} + +static bool pendingbundleMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + + psMetadata *where = psMetadataAlloc(); + // pxdqstatsGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstatsRun.dqstats_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "dqstatsRun.label", "=="); + + psString query = pxDataGet("dqstatstool_get_run.sql"); // query + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retrieve sql statement (dqstatstool_get_run.sql)"); + return(false); + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); // where whereClause + + psArray *output = p_psDBFetchResult(config->dbh); // where whereClause output + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("dqstatstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "dqstatsPendingBundle", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return(true); +} +//static bool cleanbundleMode(pxConfig *config) { +// return(true); +//} + + +static bool createbundleMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + bool status = true; + PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", true, 0); + PXOPT_LOOKUP_STR(bundleUri, config->args, "-uri", true, 0); + + // Determine the information about this run. + psString runQuery = pxDataGet("dqstatstool_get_run.sql"); + if (!runQuery) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retrieve sql statement (dqstatstool_get_run.sql)"); + return(false); + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); // Save for content query + psStringAppend(&runQuery, " AND %s", whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, runQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(runQuery); + psFree(whereClause); + return(false); + } + psFree(runQuery); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + return(false); + } + + if (psArrayLength(output) != 1) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool: single return row not found."); + psFree(output); + psFree(whereClause); + return(false); + } + + psMetadata *run = output->data[0]; + if (strcmp(psMetadataLookupStr(&status,run,"state"),"new") != 0) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool: bundle already created for this run (should not happen)."); + psFree(output); + psFree(whereClause); + return(false); + } + psFree(output); // This held the information about the Run. + + // Read column list from config file + psMetadata *recipe = psMetadataLookupPtr(&status, config->modules->recipes, "DQSTATS"); + psMetadata *columns = psMetadataLookupMetadata(&status,recipe,"STATS.COLUMNS"); + if (!columns) { + psError(PS_ERR_UNKNOWN, false, "Could not read STATS.COLUMN metadata"); + psFree(whereClause); + return(false); + } + + psMetadataIterator *iter = psMetadataIteratorAlloc(columns, PS_LIST_HEAD,NULL); + + psMetadataItem *colItem = NULL; + psString columnlist = NULL; + + psMetadata *colNames = psMetadataAlloc(); + psMetadata *colTypes = psMetadataAlloc(); + while ((colItem = psMetadataGetAndIncrement(iter))) { + if (strcmp(colItem->name, "COLUMN")) continue; + if (colItem->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool COLUMN not of type METADATA\n"); + return(NULL); + } + + bool status; + psMetadata *coldef = colItem->data.md; + + if (columnlist) { // Stick intervening commas + psStringAppend(&columnlist, ", "); + } + + psStringAppend(&columnlist, "%s.%s", + psMetadataLookupStr(&status,coldef,"TABLE"), + psMetadataLookupStr(&status,coldef,"ENTRY")); + + psMetadataAddStr(colNames,PS_LIST_TAIL, + psMetadataLookupStr(&status,coldef,"ENTRY"),0,"", + psMetadataLookupStr(&status,coldef,"COLNAME")); + psMetadataAddStr(colTypes,PS_LIST_TAIL, + psMetadataLookupStr(&status,coldef,"ENTRY"),0,"", + psMetadataLookupStr(&status,coldef,"DATATYPE")); + } + psFree(iter); + + if (!columnlist) { + psError(PS_ERR_UNKNOWN, false, "Failed to find a columnlist"); + psFree(whereClause); + psFree(colNames); + psFree(colTypes); + return(false); + } + + // Read decision engine settings from config file. + psMetadata *engine = psMetadataLookupMetadata(&status,recipe,"DECISION.ENGINE"); + if (!engine) { + psError(PS_ERR_UNKNOWN, false, "Could not read DECISION.ENGINE metadata"); + psFree(whereClause); + return(false); + } + psArray *decisionRules = psArrayAllocEmpty(4); // list of screened and verified rules. + psMetadataIterator *iterEngine = psMetadataIteratorAlloc(engine, PS_LIST_HEAD,NULL); + psMetadataItem *engineItem = NULL; + + while ((engineItem = psMetadataGetAndIncrement(iterEngine))) { + if (strcmp(engineItem->name, "RULE")) continue; + if (engineItem->type != PS_DATA_METADATA) { + psError(PS_ERR_UNKNOWN, false, "dqstatstool RULE not of type METADATA\n"); + return(NULL); + } + + bool status; + psMetadata *enginedef = engineItem->data.md; + + if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& + isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM"))) { + psArrayAdd(decisionRules,decisionRules->n,enginedef); + } + else { + psError(PS_ERR_UNKNOWN, false, "Rule invalid."); + } + } + psFree(iterEngine); + + if (decisionRules->n < 1) { + psError(PS_ERR_UNKNOWN, false, "Failed to generate a decision engine"); + psFree(decisionRules); + psFree(engine); + return(false); + } + psFree(engine); + + // Find the contents that comprise this run + psString contentQuery = pxDataGet("dqstatstool_get_contents.sql"); + if (!contentQuery) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retrieve sql statement (dqstatstool_get_contents.sql)"); + psFree(whereClause); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + return(false); + } + + psStringAppend(&contentQuery, " AND %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, contentQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contentQuery); + return(false); + } + psFree(contentQuery); + + psArray *contents = p_psDBFetchResult(config->dbh); + if (!contents) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + return(false); + } + if (!psArrayLength(contents)) { + psWarning("dqstatstool No rows found"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contents); + return(false); + } + + // Grab data from the database for a row of the bundle. + psString query = pxDataGet("dqstatstool_createbundle.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "Failed to retrieve sql statement"); + psFree(colNames); + psFree(colTypes); + psFree(columnlist); + psFree(contents); + return(false); + } + // From disttool.c: + // "psStringSubstitute fails unless the input is a psString which it determines by + // comparing the memory blocks free function to an expected value. + // pxDataGet uses psSlurp which leaves a different free function on the memory block. + // To work around this make a copy of the query before doing the substitution." + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; + psStringSubstitute(&query,columnlist, "@COLLIST@"); + psFree(columnlist); + + psArray *outTable = psArrayAllocEmpty(contents->n); + for (long i = 0; i < contents->n; i++) { // Loop over content entries. + psMetadata *row = contents->data[i]; + + // Limit it just to the stuff we care about for this row. + bool status; + psString MYquery = psStringCopy(query); + if (!MYquery) { + psTrace("debug",4,"Failed to copy, trying again."); + MYquery = psStringCopy(query); + } + + psMetadata *where = psMetadataAlloc(); + + psMetadataAddS64(where,PS_LIST_TAIL, "rawExp.exp_id", 0, "==", psMetadataLookupS64(&status,row,"exp_id")); + psMetadataAddS64(where,PS_LIST_TAIL, "chipRun.chip_id", 0, "==", psMetadataLookupS64(&status,row,"chip_id")); + psMetadataAddS64(where,PS_LIST_TAIL, "camRun.cam_id", 0, "==", psMetadataLookupS64(&status,row,"cam_id")); + // This may not be real, so we need to check first. Is there a better way to check for a null here? + if (psMetadataLookupS64(&status,row,"warp_id") && (psMetadataLookupS64(&status,row,"warp_id") != PS_MAX_S64)) { + psMetadataAddS64(where,PS_LIST_TAIL, "warpRun.warp_id", 0, "==", psMetadataLookupS64(&status,row,"warp_id")); + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (!whereClause) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate Where Condition"); + return(false); + } + psStringAppend(&MYquery, " WHERE %s", whereClause); + +/* psTrace("debug",2,"%ld %ld %ld %ld %s\n",psMetadataLookupS64(&status,row,"exp_id"), */ +/* psMetadataLookupS64(&status,row,"chip_id"), */ +/* psMetadataLookupS64(&status,row,"cam_id"), */ +/* psMetadataLookupS64(&status,row,"warp_id"), */ +/* whereClause); */ + +/* psFree(whereClause); */ +/* psFree(where); */ + // printf("%s\n",MYquery); + + if (!p_psDBRunQuery(config->dbh, MYquery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(MYquery); + psFree(query); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(outTable); + return(false); + } + + psArray *statsoutput = p_psDBFetchResult(config->dbh); + if (!statsoutput) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + return(false); + } + if (!psArrayLength(statsoutput)) { + psWarning("dqstatstool No stats rows found: %ld %s %s",i,MYquery,query); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + return(false); + } + psFree(MYquery); + psFree(whereClause); + psFree(where); + + // Magic happens and the fits table is filled. + psMetadata *statsrow = statsoutput->data[0]; // Because there should only be one row. + + psMetadataIterator *statIter = psMetadataIteratorAlloc(statsrow,PS_LIST_HEAD,NULL); + + psMetadataItem *rowItem = NULL; + psMetadata *tableRow = psMetadataAlloc(); + // set args -dbname gpc1 -createbundle -dqstats_id 2 -uri /tmp/czwtest.fits + while ((rowItem = psMetadataGetAndIncrement(statIter))) { + bool success = false; + + psString colType = psMetadataLookupStr(&status,colTypes,rowItem->name); + if (!colType) { + psError(PS_ERR_UNKNOWN,false, "Could not find type for %s",rowItem->name); + return(false); + } + psString colName = psMetadataLookupStr(&status,colNames,rowItem->name); + if (!colName) { + psError(PS_ERR_UNKNOWN,false, "Could not find name for %s",rowItem->name); + return(false); + } + + if (!strcmp(colType,"PS_DATA_BOOL")) { + success = psMetadataAddBool(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.B); + } + else if (!strcmp(colType,"PS_DATA_S8")) { + success = psMetadataAddS8(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S8); + } + else if (!strcmp(colType,"PS_DATA_S16")) { + success = psMetadataAddS16(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S16); + } + else if (!strcmp(colType,"PS_DATA_S32")) { + success = psMetadataAddS32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S32); + } + else if (!strcmp(colType,"PS_DATA_S64")) { + success = psMetadataAddS64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.S64); + } + else if (!strcmp(colType,"PS_DATA_U8")) { + success = psMetadataAddU8(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U8); + } + else if (!strcmp(colType,"PS_DATA_U16")) { + success = psMetadataAddU16(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U16); + } + else if (!strcmp(colType,"PS_DATA_U32")) { + success = psMetadataAddU32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U32); + } + else if (!strcmp(colType,"PS_DATA_U64")) { + success = psMetadataAddU64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.U64); + } + else if (!strcmp(colType,"PS_DATA_F32")) { + success = psMetadataAddF32(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.F32); + } + else if (!strcmp(colType,"PS_DATA_F64")) { + success = psMetadataAddF64(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.F64); + } + else if (!strcmp(colType,"PS_DATA_STRING")) { + success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.str); + } + else if (!strcmp(colType,"PS_DATA_TIME")) { + psString timeString = psTimeToISO(rowItem->data.V); + success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",timeString); + psFree(timeString); + } + + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error constructing fits table row. %s %s",colType,colName); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + psFree(colType); + psFree(colName); + return(false); + } + } + // Calculate if this exposure was "good" or not. + bool accept = true; + for (long j = 0; j < decisionRules->n; j++) { + psMetadata *rule = decisionRules->data[j]; + bool status = true; + psString colname = psMetadataLookupStr(&status,rule,"COLNAME"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column name"); + return(false); + } + psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); + return(false); + } + psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); + return(false); + } + // Not happy with this being set to a F32. Can this ever be something else? + psF32 value = psMetadataLookupF32(&status,tableRow,colname); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f %f %f) in compare with rule.",colname,min_value,max_value,value); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(tableRow); + return(false); + } + if ((value > max_value)||(value < min_value)) { + accept = false; + } + } + bool success; + success = psMetadataAddBool(tableRow,PS_LIST_TAIL,"ACCEPT",0,"",accept); + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error adding exposure quality to fits table row."); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + return(false); + } + + // Add the row to the table array. + psArrayAdd(outTable,0,tableRow); + psFree(tableRow); + } + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + // Define the fits table here. + + psFits *Table = psFitsOpen(bundleUri,"w"); + if (!Table) { + psError(PS_ERR_UNKNOWN, false, "Could not open the fits table for writing."); + psFree(outTable); + return(false); + } + + psMetadata *header = psMetadataAlloc(); + psMetadataAddS64(header, PS_LIST_TAIL, "DQSTATS_ID", 0, "ID number of DQSTATS run", dqstats_id); + + if (!psFitsWriteTable(Table,header,outTable,"DQSTATS")) { + psError(PS_ERR_UNKNOWN, false, "Unable to write table to fits file."); + psFree(outTable); + psFree(Table); + psFree(header); + return(false); + } + psFree(outTable); + psFree(header); + + // Tear down, cleanup. + psFitsClose(Table); + return(true); +} + + +static bool updaterunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + if (state && ! pxIsValidState(state)) { + psError(PXTOOLS_ERR_CONFIG, false, "pxIsValidState failed"); + return(false); + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + PXOPT_COPY_S64(config->args, where, "-state", "state", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return(false); + } + + psString query = psStringCopy("UPDATE dqstatsRun "); + psStringAppend(&query," SET state = '%s' ", state); + if (fault) { + psStringAppend(&query, ", fault = %d ", fault); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query," WHERE %s ",whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + psFree(query); + psFree(where); + psFree(whereClause); + + return(true); +} + +static bool revertrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config,false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dqstats_id", "dqstats_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return(false); + } + + psString query = psStringCopy("UPDATE dqstatsRun SET fault = 0 "); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query," WHERE fault != 0 AND %s ",whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + psFree(query); + psFree(whereClause); + psFree(where); + + return(true); +} + + + + + + + Index: branches/simtest_nebulous_branches/ippTools/src/dqstatstool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dqstatstool.h (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/dqstatstool.h (revision 27840) @@ -0,0 +1,18 @@ +#ifndef DQSTATS_H +#define DQSTATS_H 1 + +#include "pxtools.h" + +typedef enum { + DQSTATS_MODE_NONE = 0x0, + DQSTATS_MODE_DEFINEBYQUERY, + DQSTATS_MODE_PENDINGBUNDLE, + DQSTATS_MODE_CREATEBUNDLE, + DQSTATS_MODE_CLEANBUNDLE, + DQSTATS_MODE_UPDATERUN, + DQSTATS_MODE_REVERTRUN, +} dqstatstoolMode; + +pxConfig *dqstatstoolConfig(pxConfig *config, int argc, char **argv); + +#endif Index: branches/simtest_nebulous_branches/ippTools/src/dqstatstoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/dqstatstoolConfig.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/dqstatstoolConfig.c (revision 27840) @@ -0,0 +1,100 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include "pxtools.h" +#include "pxdqstats.h" +#include "dqstatstool.h" + +pxConfig *dqstatstoolConfig(pxConfig *config, int argc, char **argv) { + + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, "DQSTATS"); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return(NULL); + } + psTime *now = psTimeGetNow(PS_TIME_TAI); + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "set run label", NULL); + + pxdqstatsSetSearchArgs(definebyqueryArgs); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-force", 0, "invalidate old entries that match", false); + psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit bundle size to N rows", 0); + + // -createbundle + psMetadata *createbundleArgs = psMetadataAlloc(); + psMetadataAddS64(createbundleArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddStr(createbundleArgs, PS_LIST_TAIL, "-uri", 0, "output table URI", 0); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by dqstatsRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set dqstatsRun stats", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -pendingbundle + psMetadata *pendingbundleArgs = psMetadataAlloc(); + psMetadataAddS64(pendingbundleArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + psMetadataAddU64(pendingbundleArgs, PS_LIST_TAIL, "-limit", 0, "limit number of bundles to N", 0); + psMetadataAddBool(pendingbundleArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(pendingbundleArgs, PS_LIST_TAIL, "-label", 0, "search by dqstatsRun.label", NULL); + + // -revertrun + psMetadata *revertrunArgs = psMetadataAlloc(); + psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label", 0, "search by dqstatsRun.label", NULL); + psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-dqstats_id", 0, "search by dqstats_id", 0); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", DQSTATS_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change dqstats run properties", DQSTATS_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-createbundle", "create the bundle for a run", DQSTATS_MODE_CREATEBUNDLE, createbundleArgs); + PXOPT_ADD_MODE("-pendingbundle", "list bundles needing creation", DQSTATS_MODE_PENDINGBUNDLE, pendingbundleArgs); + PXOPT_ADD_MODE("-revertrun", "revert runs with errors", DQSTATS_MODE_REVERTRUN, revertrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return(NULL); + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return(NULL); + } + + // config = psMetadataConfigRead(config,1,"dqstatsTool.config",1); + + return config; +} + + + + Index: branches/simtest_nebulous_branches/ippTools/src/faketool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/faketool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/faketool.c (revision 27840) @@ -52,4 +52,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -91,4 +92,5 @@ MODECASE(FAKETOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(FAKETOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(FAKETOOL_MODE_IMPORTRUN, importrunMode); @@ -121,4 +123,6 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); @@ -128,5 +132,5 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); // XXX does this choose the right label? PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); @@ -164,5 +168,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -175,4 +179,7 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -183,5 +190,5 @@ psString query = pxDataGet("faketool_find_camrun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -263,5 +270,5 @@ // queue the exp - if (!pxfakeQueueByCamID(config, cam_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!pxfakeQueueByCamID(config, cam_id, workdir, label, data_group ? data_group : label, dist_group, reduction, expgroup, dvodb, tess_id, end_stage, note)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -328,41 +335,26 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group", "fakeRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "fakeRun.dist_group", "=="); + + if (!psListLength(where->list)) { psFree(where); where = NULL; - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set fakeRun.state to state - if (!pxfakeRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set fakeRun.label to label - if (!pxfakeRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "fakeRun", "fake_id", "fakeProcessedImfile", true); + if (!result) { + psError(psErrorCodeLast(), false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -383,5 +375,5 @@ psString query = pxDataGet("faketool_find_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -450,5 +442,5 @@ psString query = pxDataGet("faketool_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -566,15 +558,15 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.telescope", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); psString query = pxDataGet("faketool_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "fakeProcessedImfile"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -675,5 +667,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -681,5 +673,5 @@ psString query = pxDataGet("faketool_revertprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -822,5 +814,5 @@ psString query = pxDataGet("faketool_unmasked.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -899,5 +891,5 @@ psString query = pxDataGet("faketool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -964,5 +956,5 @@ psString query = pxDataGet("faketool_pendingcleanupimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1025,5 +1017,5 @@ psString query = pxDataGet("faketool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1087,5 +1079,5 @@ psString query = pxDataGet("faketool_completely_processed_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1160,8 +1152,11 @@ fakeRun->workdir, fakeRun->label, + fakeRun->data_group, + fakeRun->dist_group, fakeRun->dvodb, fakeRun->tess_id, fakeRun->reduction, - fakeRun->end_stage + fakeRun->end_stage, + NULL // note does not propagate )) { psError(PS_ERR_UNKNOWN, false, "failed to queue warpRun"); @@ -1251,4 +1246,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) @@ -1273,5 +1272,8 @@ return false; } - + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); @@ -1285,5 +1287,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1354,7 +1356,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "fakeRun"); psAssert (item, "entry not in input?"); Index: branches/simtest_nebulous_branches/ippTools/src/faketool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/faketool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/faketool.h (revision 27840) @@ -45,4 +45,5 @@ FAKETOOL_MODE_TOFULLIMFILE, FAKETOOL_MODE_TOPURGEDIMFILE, + FAKETOOL_MODE_TOSCRUBBEDIMFILE, FAKETOOL_MODE_EXPORTRUN, FAKETOOL_MODE_IMPORTRUN Index: branches/simtest_nebulous_branches/ippTools/src/faketoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/faketoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/faketoolConfig.c (revision 27840) @@ -39,5 +39,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -47,4 +47,6 @@ psMetadata *queueArgs = psMetadataAlloc(); // XXX need to allow multiple exp_ids + psMetadataAddS64(queueArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(queueArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); psMetadataAddS64(queueArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); @@ -93,4 +95,7 @@ psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -139,9 +144,14 @@ psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by data_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -pendingexp @@ -301,4 +311,9 @@ psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "fake ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -337,4 +352,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", FAKETOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", FAKETOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", FAKETOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/simtest_nebulous_branches/ippTools/src/flatcorr.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/flatcorr.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/flatcorr.c (revision 27840) @@ -33,4 +33,6 @@ #include "pxtools.h" #include "pxchip.h" +#include "pxcam.h" +#include "pxadd.h" #include "flatcorr.h" @@ -38,7 +40,9 @@ static bool definerunMode(pxConfig *config); static bool addchipMode(pxConfig *config); +static bool dropchipMode(pxConfig *config); static bool addcameraMode(pxConfig *config); -static bool dropchipMode(pxConfig *config); static bool dropcameraMode(pxConfig *config); +static bool advancecameraMode(pxConfig *config); +static bool advanceaddstarMode(pxConfig *config); static bool pendingprocessMode(pxConfig *config); static bool addprocessMode(pxConfig *config); @@ -46,6 +50,4 @@ static bool inputexpMode(pxConfig *config); static bool inputimfileMode(pxConfig *config); -static bool exportrunMode(pxConfig *config); -static bool importrunMode(pxConfig *config); static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state); @@ -72,7 +74,9 @@ MODECASE(FLATCORR_MODE_DEFINERUN, definerunMode); MODECASE(FLATCORR_MODE_ADDCHIP, addchipMode); + MODECASE(FLATCORR_MODE_DROPCHIP, dropchipMode); MODECASE(FLATCORR_MODE_ADDCAMERA, addcameraMode); - MODECASE(FLATCORR_MODE_DROPCHIP, dropchipMode); MODECASE(FLATCORR_MODE_DROPCAMERA, dropcameraMode); + MODECASE(FLATCORR_MODE_ADVANCECAMERA, advancecameraMode); + MODECASE(FLATCORR_MODE_ADVANCEADDSTAR, advanceaddstarMode); MODECASE(FLATCORR_MODE_PENDINGPROCESS, pendingprocessMode); MODECASE(FLATCORR_MODE_ADDPROCESS, addprocessMode); @@ -80,6 +84,4 @@ MODECASE(FLATCORR_MODE_INPUTEXP, inputexpMode); MODECASE(FLATCORR_MODE_INPUTIMFILE, inputimfileMode); - MODECASE(FLATCORR_MODE_EXPORTRUN, exportrunMode); - MODECASE(FLATCORR_MODE_IMPORTRUN, importrunMode); default: psAbort("invalid option (this should not happen)"); @@ -113,9 +115,9 @@ if (!psListLength(where->list)) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - // require the camera to be defined: this analysis does not make sense + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // require the camera to be defined: this analysis does not make sense // across multiple cameras PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); @@ -131,4 +133,6 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + PXOPT_LOOKUP_BOOL(make_correction, config->args, "-make_correction", false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -138,5 +142,5 @@ psString query = pxDataGet("chiptool_find_rawexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -169,12 +173,12 @@ if (pretend) { - for (long i = 0; i < psArrayLength(output); i++) { - // negative simple so the default is true - if (!ippdbPrintMetadataRaw(stdout, output->data[i], !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } + for (long i = 0; i < psArrayLength(output); i++) { + // negative simple so the default is true + if (!ippdbPrintMetadataRaw(stdout, output->data[i], !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } psFree(output); return true; @@ -191,19 +195,20 @@ // create a new flatcorrRun if (!flatcorrRunInsert( - config->dbh, + config->dbh, 0, // corr_id - det_type, + det_type, dvodb, - camera, - telescope, - NULL, + camera, + telescope, + NULL, filter, "reg", // state + make_correction, workdir, label, reduction, - region, - NULL, - 0 + region, + NULL, + 0 )) { if (!psDBRollback(config->dbh)) { @@ -233,5 +238,5 @@ // queue the exp : force this to stop at the chip stage - psS64 chip_id = pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, "chip"); + psS64 chip_id = pxchipQueueByExpTag(config, exp_id, workdir, label, label, NULL, reduction, expgroup, dvodb, tess_id, "chip", note); if (!chip_id) { if (!psDBRollback(config->dbh)) { @@ -291,4 +296,5 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + PXOPT_LOOKUP_BOOL(make_correction, config->args, "-make_correction", false); // XXX probably should make the region in -set_region match ra_min, ra_max, etc @@ -297,25 +303,26 @@ // start a transaction so we don't end up with an exp without any associted // imfiles - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + // if (!psDBTransaction(config->dbh)) { + // psError(PS_ERR_UNKNOWN, false, "database error"); + // return false; + // } // create a new flatcorrRun flatcorrRunRow *row = flatcorrRunRowAlloc( - 0, // corr_id - det_type, - dvodb, - camera, - telescope, - NULL, - filter, - "reg", // state - workdir, - label, - reduction, - region, - NULL, // hostname - 0 // fault + 0, // corr_id + det_type, + dvodb, + camera, + telescope, + NULL, + filter, + "reg", // state + make_correction, + workdir, + label, + reduction, + region, + NULL, // hostname + 0 // fault ); @@ -328,9 +335,14 @@ return false; } - + // figure out the ID of the flatcorrRun we just created psS64 corr_id = psDBLastInsertID(config->dbh); row->corr_id = corr_id; + // if (!psDBCommit(config->dbh)) { + // psError(PS_ERR_UNKNOWN, false, "database error"); + // return false; + // } + flatcorrRunPrintObject (stdout, row, !simple); return true; @@ -365,9 +377,52 @@ psString query = pxDataGet("flatcorr_dropchip.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; } if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} + +static bool addcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + + // add a flatcorrCamLink (initial state has include = TRUE) + // XXX should add checks that the chip_id and corr_id are in ChipLink + if (!flatcorrCamLinkInsert(config->dbh, corr_id, chip_id, cam_id, 1)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool dropcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + + // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld + psString query = pxDataGet("flatcorr_dropcamera.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) cam_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -380,5 +435,5 @@ // select the flatcorr chip runs that have completed and for which there is no camera entry // queue a new camera run for them -static bool addcameraMode(pxConfig *config) +static bool advancecameraMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -388,15 +443,25 @@ PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "flatcorrRun.label", "=="); + psString query = pxDataGet("flatcorr_chiprundone.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -459,13 +524,18 @@ // queue the exp if (!pxcamQueueByChipID( - config, - row->chip_id, - row->workdir, - row->label, - row->reduction, - row->expgroup, - row->dvodb, - row->tess_id, - "camera")) { + config, + row->chip_id, + row->workdir, + row->label, + row->data_group, + row->dist_group, + row->reduction, + row->expgroup, + row->dvodb, + row->tess_id, + "camera", + row->magicked, + NULL // note does not propragate + )) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -478,8 +548,8 @@ } - // figure out the ID of the flatcorrRun we just created - psS64 cam_id = psDBLastInsertID(config->dbh); - - // add the camRun entry to the flatcorrCamLink table (include is TRUE) + // figure out the ID of the flatcorrRun we just created + psS64 cam_id = psDBLastInsertID(config->dbh); + + // add the camRun entry to the flatcorrCamLink table (include is TRUE) if (!flatcorrCamLinkInsert(config->dbh, corr_id, row->chip_id, cam_id, 1)) { if (!psDBRollback(config->dbh)) { @@ -489,5 +559,5 @@ return false; } - + psFree(row); } @@ -502,31 +572,7 @@ } -static bool dropcameraMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); - - // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld - psString query = pxDataGet("flatcorr_dropcamera.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) cam_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - - return true; -} - -// select the flatcorr chip runs that have completed and for which there is no camera entry -// queue a new camera run for them -static bool pendingprocessMode(pxConfig *config) +// Select the flatcorr camera runs that have completed and for which there is no addstar +// entry. Queue a new addstar run for them +static bool advanceaddstarMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -534,12 +580,13 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - psString query = pxDataGet("flatcorr_pendingprocess.sql"); + pxAddLabelSearchArgs (config, where, "-label", "flatcorrRun.label", "=="); + + psString query = pxDataGet("flatcorr_camerarundone.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; } @@ -553,7 +600,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -575,8 +622,177 @@ } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "flatcorr_addcamera", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of chipRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 corr_id = psMetadataLookupS64(&status, md, "corr_id"); + if (!status) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for corr_id"); + psFree(output); + return false; + } + + camRunRow *row = camRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp : force image_only to be false (flatcorr is meaningless with just image info) + if (!pxaddQueueByCamID( + config, + row->cam_id, + row->workdir, + row->reduction, + row->label, + row->data_group, + row->dvodb, + NULL, // note is not propagated + 0)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue cam_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 add_id = psDBLastInsertID(config->dbh); + + // add the addRun entry to the flatcorrAddstarLink table (include is TRUE) + if (!flatcorrAddstarLinkInsert(config->dbh, corr_id, row->cam_id, add_id, 1)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return false; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool pendingprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("flatcorr_pendingprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + return true; +} + +// XXX need a fault state +static bool addprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, hostname, code, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); + return false; + } + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (!setflatcorrRunState(config, corr_id, state)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set run state"); + return false; } @@ -598,7 +814,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -621,7 +837,7 @@ if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; } @@ -638,5 +854,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); - + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "=="); @@ -644,5 +860,5 @@ psString query = pxDataGet("flatcorr_inputimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -657,7 +873,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -680,43 +896,6 @@ if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - return true; -} - -// XXX need a fault state -static bool addprocessMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - - char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, hostname, code, corr_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); - return false; - } - - return true; -} - -static bool updaterunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (!setflatcorrRunState(config, corr_id, state)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to set run state"); + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); return false; } @@ -731,7 +910,7 @@ // check that state is a valid string value - if (!strcmp(state, "reg") && - !strcmp(state, "new") && - !strcmp(state, "full")) + if (!strcmp(state, "reg") && + !strcmp(state, "new") && + !strcmp(state, "full")) { psError(PS_ERR_UNKNOWN, false, "invalid state: %s", state); @@ -748,144 +927,2 @@ return true; } - -bool exportrunMode(pxConfig *config) -{ - typedef struct ExportTable { - char tableName[80]; - char sqlFilename[80]; - } ExportTable; - - int numExportTables = 3; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(det_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - - FILE *f = fopen (outfile, "w"); - if (f == NULL) { - psError(PS_ERR_UNKNOWN, false, "failed to open output file"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-corr_id", "corr_id", "=="); - - ExportTable tables [] = { - {"flatcorrRun", "flatcorr_export_run.sql"}, - {"flatcorrCamLink", "flatcorr_export_cam_link.sql"}, - {"flatcorrChipLink", "flatcorr_export_chip_link.sql"}, - }; - - for (int i=0; i < numExportTables; i++) { - psString query = pxDataGet(tables[i].sqlFilename); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("regtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // we must write the export table in non-simple (true) format - if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - psFree(output); - } - - fclose (f); - - return true; -} - -bool importrunMode(pxConfig *config) -{ - unsigned int nFail; - psMetadataItem *item, *entry; - - int numImportTables = 3; - - char tables[3] [80] = {"flatcorrRun", "flatcorrCamLink", "flatcorrChipLink"}; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); - - psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - - fprintf (stdout, "---- input ----\n"); - psMetadataPrint (stderr, input, 1); - - for (int i = 0; i < numImportTables; i++) { - item = psMetadataLookup (input, tables[i]); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - entry = psListGet (item->data.list, 0); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - - switch (i) { - case 0: - { - flatcorrRunRow *flatcorrRun = flatcorrRunObjectFromMetadata (entry->data.md); - flatcorrRunInsertObject (config->dbh, flatcorrRun); - - // fprintf (stdout, "---- flatcorr run ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - case 1: - { - flatcorrCamLinkRow *flatcorrCamLink = flatcorrCamLinkObjectFromMetadata (entry->data.md); - flatcorrCamLinkInsertObject (config->dbh, flatcorrCamLink); - - // fprintf (stdout, "---- flatcorr cam link ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - case 2: - { - flatcorrChipLinkRow *flatcorrChipLink = flatcorrChipLinkObjectFromMetadata (entry->data.md); - flatcorrChipLinkInsertObject (config->dbh, flatcorrChipLink); - - // fprintf (stdout, "---- flatcorr chip link ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - } - } - return true; -} Index: branches/simtest_nebulous_branches/ippTools/src/flatcorr.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/flatcorr.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/flatcorr.h (revision 27840) @@ -28,7 +28,9 @@ FLATCORR_MODE_DEFINERUN, FLATCORR_MODE_ADDCHIP, + FLATCORR_MODE_DROPCHIP, FLATCORR_MODE_ADDCAMERA, - FLATCORR_MODE_DROPCHIP, FLATCORR_MODE_DROPCAMERA, + FLATCORR_MODE_ADVANCECAMERA, + FLATCORR_MODE_ADVANCEADDSTAR, FLATCORR_MODE_PENDINGPROCESS, FLATCORR_MODE_ADDPROCESS, @@ -36,6 +38,4 @@ FLATCORR_MODE_INPUTEXP, FLATCORR_MODE_INPUTIMFILE, - FLATCORR_MODE_EXPORTRUN, - FLATCORR_MODE_IMPORTRUN } flatcorrMode; Index: branches/simtest_nebulous_branches/ippTools/src/flatcorrConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/flatcorrConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/flatcorrConfig.c (revision 27840) @@ -59,4 +59,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessalation", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_region", 0, "define region", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-make_correction", 0, "generate a correction image and add to detrend database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); @@ -65,4 +66,6 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-inst", 0, "define camera", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-det_type", 0, "define detrend type to be generated", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); @@ -74,4 +77,5 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessalation", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_region", 0, "define region", NULL); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-make_correction", 0, "generate a correction image and add to detrend database", false); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -88,7 +92,7 @@ // -addcamera psMetadata *addcameraArgs = psMetadataAlloc(); - psMetadataAddU64 (addcameraArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-chip_id", 0, "define Chip ID (required)", 0); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-cam_id", 0, "define Camera ID (required)", 0); // -dropcamera @@ -96,4 +100,18 @@ psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-cam_id", 0, "define Camera ID (required)", 0); + + // -advancecamera + psMetadata *advancecameraArgs = psMetadataAlloc(); + psMetadataAddU64 (advancecameraArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(advancecameraArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(advancecameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddStr(advancecameraArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + + // -advanceaddstar + psMetadata *advanceaddstarArgs = psMetadataAlloc(); + psMetadataAddU64 (advanceaddstarArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(advanceaddstarArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(advanceaddstarArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddStr(advanceaddstarArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); // -pendingprocess @@ -113,4 +131,5 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-corr_id", 0, "define correction id (required)", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + // XXX add mechanism to change this value: psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-set_correction", 0, "generate a correction image and add to detrend database", false); // -inputexp @@ -142,17 +161,17 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "create a new, populated flat correction run", FLATCORR_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "create a new, empty flat correction run", FLATCORR_MODE_DEFINERUN, definerunArgs); - PXOPT_ADD_MODE("-addchip", "add a chip to a flat correction run", FLATCORR_MODE_ADDCHIP, addchipArgs); - PXOPT_ADD_MODE("-addcamera", "migrate completed chips to camera stage analysis", FLATCORR_MODE_ADDCAMERA, addcameraArgs); - PXOPT_ADD_MODE("-dropchip", "drop a chip from a flat correction run", FLATCORR_MODE_DROPCHIP, dropchipArgs); - PXOPT_ADD_MODE("-dropcamera", "drop an exposure (camera stage analysis)", FLATCORR_MODE_DROPCAMERA, dropcameraArgs); - PXOPT_ADD_MODE("-pendingprocess", "show flat correction runs needing to be processed", FLATCORR_MODE_PENDINGPROCESS, pendingprocessArgs); - PXOPT_ADD_MODE("-addprocess", "report completed flat correction analysis", FLATCORR_MODE_ADDPROCESS, addprocessArgs); - PXOPT_ADD_MODE("-updaterun", "change a flat calibration run's state", FLATCORR_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-inputexp", "list exposures for a correction run", FLATCORR_MODE_INPUTEXP, inputexpArgs); - PXOPT_ADD_MODE("-inputimfile", "list imfiles for a chip run", FLATCORR_MODE_INPUTIMFILE, inputimfileArgs); - PXOPT_ADD_MODE("-exportrun", "export run for import on other database", FLATCORR_MODE_EXPORTRUN, exportrunArgs); - PXOPT_ADD_MODE("-importrun", "import run from metadata file", FLATCORR_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-definebyquery", "create a new, populated flat correction run", FLATCORR_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "create a new, empty flat correction run", FLATCORR_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-addchip", "add an existing chip run to a flat correction run", FLATCORR_MODE_ADDCHIP, addchipArgs); + PXOPT_ADD_MODE("-addcamera", "add an existing camera run to a flat correction run", FLATCORR_MODE_ADDCAMERA, addcameraArgs); + PXOPT_ADD_MODE("-advancecamera", "migrate completed chips to camera stage analysis", FLATCORR_MODE_ADVANCECAMERA, advancecameraArgs); + PXOPT_ADD_MODE("-advanceaddstar", "migrate completed exposures to addstar processing", FLATCORR_MODE_ADVANCEADDSTAR, advanceaddstarArgs); + PXOPT_ADD_MODE("-dropchip", "drop a chip from a flat correction run", FLATCORR_MODE_DROPCHIP, dropchipArgs); + PXOPT_ADD_MODE("-dropcamera", "drop an exposure (camera stage analysis)", FLATCORR_MODE_DROPCAMERA, dropcameraArgs); + PXOPT_ADD_MODE("-pendingprocess", "show flat correction runs needing to be processed", FLATCORR_MODE_PENDINGPROCESS, pendingprocessArgs); + PXOPT_ADD_MODE("-addprocess", "report completed flat correction analysis", FLATCORR_MODE_ADDPROCESS, addprocessArgs); + PXOPT_ADD_MODE("-updaterun", "change a flat calibration run's state", FLATCORR_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-inputexp", "list exposures for a correction run", FLATCORR_MODE_INPUTEXP, inputexpArgs); + PXOPT_ADD_MODE("-inputimfile", "list imfiles for a chip run", FLATCORR_MODE_INPUTIMFILE, inputimfileArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/simtest_nebulous_branches/ippTools/src/magicdstool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/magicdstool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/magicdstool.c (revision 27840) @@ -33,17 +33,18 @@ static bool definebyqueryMode(pxConfig *config); -static psS64 definerunMode(pxConfig *config); +static bool definecopyMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool todestreakMode(pxConfig *config); static bool adddestreakedfileMode(pxConfig *config); +static bool advancerunMode(pxConfig *config); static bool revertdestreakedfileMode(pxConfig *config); +static bool clearstatefaultsMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); static bool toremoveMode(pxConfig *config); -static bool torestoreMode(pxConfig *config); static bool torevertMode(pxConfig *config); - -static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); -static bool magicDSRunComplete(pxConfig *config, bool setmagicked); -static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id); +static bool completedrevertMode(pxConfig *config); +static bool tocleanupMode(pxConfig *config); + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state); # define MODECASE(caseName, func) \ @@ -66,13 +67,16 @@ switch (config->mode) { MODECASE(MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(MAGICDSTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(MAGICDSTOOL_MODE_DEFINECOPY, definecopyMode); MODECASE(MAGICDSTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(MAGICDSTOOL_MODE_TODESTREAK, todestreakMode); MODECASE(MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileMode); + MODECASE(MAGICDSTOOL_MODE_ADVANCERUN, advancerunMode); MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); + MODECASE(MAGICDSTOOL_MODE_CLEARSTATEFAULTS, clearstatefaultsMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); MODECASE(MAGICDSTOOL_MODE_TOREMOVE, toremoveMode); - MODECASE(MAGICDSTOOL_MODE_TORESTORE, torestoreMode); MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); + MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); + MODECASE(MAGICDSTOOL_MODE_TOCLEANUP, tocleanupMode); default: psAbort("invalid option (this should not happen)"); @@ -105,8 +109,10 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); - PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); - PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -119,11 +125,11 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id","magicRun.magic_id", "=="); PXOPT_COPY_S32(config->args, where, "-streaks_max","streaks", "<="); - pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); // define using magicRun label ippStage stageNum = ippStringToStage(stage); - + psString query = NULL; switch (stageNum) { @@ -145,8 +151,8 @@ case IPP_STAGE_FAKE: case IPP_STAGE_STACK: - psError(PXTOOLS_ERR_DATA, true, "%sRuns do not need to be destreaked", stage); + psError(PXTOOLS_ERR_CONFIG, true, "%sRuns do not need to be destreaked", stage); return false; case IPP_STAGE_NONE: - psError(PXTOOLS_ERR_DATA, true, "%s is not a valid stage", stage); + psError(PXTOOLS_ERR_CONFIG, true, "%s is not a valid stage", stage); return false; default: @@ -157,5 +163,5 @@ if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -231,5 +237,5 @@ // Parse the list of runs ready to be destreaked - if (!dry_run && !psDBTransaction(config->dbh)) { + if (!pretend && !psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -245,6 +251,7 @@ psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunDataGroup = psMetadataLookupStr(NULL, row, "data_group"); psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir"); - + // if workdir is not supplied use the magicRun's if (!workdir) { @@ -266,8 +273,11 @@ cam_id, set_label ? set_label : magicRunLabel, + set_data_group ? set_data_group : magicRunDataGroup, outroot, recoveryroot, - re_place, - 0); // remove + noreplace ? 0 :1, // re_place + 0, // remove + 0, // fault + note); // remove psFree(outroot); @@ -276,5 +286,5 @@ } - if (!dry_run) { + if (!pretend) { if (!magicDSRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -295,5 +305,5 @@ } - if (!dry_run && !psDBCommit(config->dbh)) { + if (!pretend && !psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -312,107 +322,306 @@ } -static psS64 definerunMode(pxConfig *config) + +// XXX This currently allows multiple destreak runs to be queued on the same exposure if there are multiple +// magicRuns selected! +static bool definecopyMode(pxConfig *config) +{ + // Required + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // Optional + PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // search args + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "chipRun.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_S32(config->args, where, "-streaks_max", "magicMask.streaks", "<="); + + pxAddLabelSearchArgs (config, where, "-magic_label", "magicRun.label", "=="); // define magic label + psString labelName = NULL; // Name of label + psStringAppend(&labelName, "%sRun.label", stage); + pxAddLabelSearchArgs (config, where, "-stage_label", labelName, "=="); // define stageRun label + psFree(labelName); + + ippStage stageNum = ippStringToStage(stage); + + psString query = NULL; + switch (stageNum) { + case IPP_STAGE_RAW: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Raw stage is not appropriate for a copied destreak"); + return false; + case IPP_STAGE_CHIP: + query = pxDataGet("magicdstool_definecopy_chip.sql"); + break; + case IPP_STAGE_WARP: + query = pxDataGet("magicdstool_definecopy_warp.sql"); + break; + case IPP_STAGE_CAMERA: + case IPP_STAGE_DIFF: + case IPP_STAGE_FAKE: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "%s has not been coded.", stage); + return false; + case IPP_STAGE_STACK: + psError(PXTOOLS_ERR_CONFIG, true, "Stacks do not need to be destreaked"); + return false; + case IPP_STAGE_NONE: + psError(PXTOOLS_ERR_CONFIG, true, "%s is not a valid stage", stage); + return false; + default: + psError(PXTOOLS_ERR_PROG, true, "ippStageToString returned %d for invalid stage %s", + stageNum, stage); + return false; + } + + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + const char *rerun_flag = rerun ? "" : "\n"; // String to give query to activate (or not) rerun + + if (stageNum != IPP_STAGE_DIFF) { + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQueryF(config->dbh, query, set_label, rerun_flag)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + } else { + // diff stage query has two types bothways and !bothways + // so we need to send the rerun flag and the where data twice + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, rerun_flag, whereString, rerun_flag, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + psFree(query); + return false; + } + psFree(whereString); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // Parse the list of runs ready to be destreaked + + if (!pretend && !psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; // Row of interest + psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); + psS64 inv_magic_id = psMetadataLookupS64(NULL, row, "inv_magic_id"); + psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); + psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunDataGroup = psMetadataLookupStr(NULL, row, "data_group"); + + psString outroot = NULL; + // set outroot to workdir/exp_id/stage for example /somewhere/424242/chip + psStringAppend(&outroot, "%s/%" PRId64 "/%s", workdir, exp_id, stage); + + // create a new magicRun for this group + magicDSRunRow *run = magicDSRunRowAlloc( + 0, // magic_ds_id + magic_id, + inv_magic_id, + "new", + stage, + stage_id, + cam_id, + set_label ? set_label : magicRunLabel, + set_data_group ? set_data_group : magicRunDataGroup, + outroot, + recoveryroot, + noreplace ? 0 :1, // re_place + 0, // remove + 0, // fault + note); // remove + + psFree(outroot); + if (!run) { + psAbort("failed to alloc magicDSRun object"); + } + + if (!pretend) { + if (!magicDSRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psS64 magic_ds_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_ds_id = magic_ds_id; + } + + psArrayAdd(list, list->n, run); + psFree(run); + } + + if (!pretend && !psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(output); + + if (!magicDSRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(list); + + return true; +} + + +static bool updaterunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + + // optional + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + psString setString = NULL; + if (set_label) { + psStringAppend(&setString, ", label = '%s'", set_label); + } + if (set_data_group) { + psStringAppend(&setString, ", data_group = '%s'", set_data_group); + } + + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); + if (magic_ds_id) { + + return setmagicDSRunState(config, magic_ds_id, setString, NULL, state); + + } else if (!strcmp(state, "full")) { + psError(PS_ERR_UNKNOWN, true, "magic_ds_id is required to update run state to full"); + return false; + } + // we can transition by query as well + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "LIKE"); + + if (psListLength(where->list) < 2) { + psError(PS_ERR_UNKNOWN, true, "at least 2 search arguments are required"); + return false; + } + + + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); + if (!noreplace) { + psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0); + } + bool result = setmagicDSRunState(config, magic_ds_id, setString, where, state); + psFree(where); + + return result; +} + + +static bool todestreakMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); - PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false); - - // optional - PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); - PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); - PXOPT_LOOKUP_BOOL(remove, config->args, "-remove", false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - - psS64 stage_id = 0, cam_id = 0; - - if (!magicDSGetIDs(config, stage, magic_id, &stage_id, &cam_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to get ids"); - return false; - } - - magicDSRunRow *run = magicDSRunRowAlloc( - 0, // ID - magic_id, - 0, // inv_magic_id - "new", // state - stage, - stage_id, - cam_id, - label, - outroot, - recoveryroot, - re_place, - remove - ); - - if (!run) { - psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object"); - return false; - } - if (!magicDSRunInsertObject(config->dbh, run)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(run); - return false; - } - - psS64 magic_ds_id = psDBLastInsertID(config->dbh); - run->magic_ds_id = magic_ds_id; - - if (!magicDSRunPrintObject(stdout, run, !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(run); - return false; - } - - psFree(run); - - return magic_id; -} - - -static bool updaterunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { - // set detRun.state to state - return setmagicDSRunState(config, magic_ds_id, state); - } - - return true; -} - - -static bool todestreakMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_todestreak.sql"); + psString sql_file = NULL; + psStringAppend(&sql_file, "magicdstool_todestreak_%s.sql", stage); + + psString query = pxDataGet(sql_file); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement from %s", sql_file); + psFree(sql_file); + return false; + } + psFree(sql_file); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -472,5 +681,5 @@ { // first query the magicDSRun to find the stage and the stage_id - psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64; + psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { @@ -499,4 +708,5 @@ psString stage= psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); ippStage stageNum = ippStringToStage(stage); @@ -526,5 +736,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -545,5 +755,5 @@ { // first query the magicDSRun to find the stage and the stage_id - psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64; + psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { @@ -572,4 +782,5 @@ psString stage = psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); ippStage stageNum = ippStringToStage(stage); @@ -597,5 +808,5 @@ return false; } - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -625,4 +836,7 @@ PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false); PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false); + PXOPT_LOOKUP_F32(streak_frac, config->args, "-streak_frac", false, false); + PXOPT_LOOKUP_F32(nondiff_frac, config->args, "-nondiff_frac", false, false); + PXOPT_LOOKUP_F32(run_time, config->args, "-run_time", false, false); if (setmagicked && (fault != 0)) { @@ -647,5 +861,15 @@ } - if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault, "full")) { + if (!magicDSFileInsert(config->dbh, + magic_ds_id, + component, + backup_path_base, + recovery_path_base, + streak_frac, + nondiff_frac, + run_time, + fault, + "full" // data_state + )) { // rollback if (!psDBRollback(config->dbh)) { @@ -656,13 +880,4 @@ } - if (!magicDSRunComplete(config, setmagicked)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -673,78 +888,26 @@ } -static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id) + +static bool advancerunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(stage, false); - PS_ASSERT_PTR_NON_NULL(stage_id, false); - PS_ASSERT_PTR_NON_NULL(cam_id, false); - - int stageNum = ippStringToStage(stage);; - if (stageNum == IPP_STAGE_NONE) { - psError(PXTOOLS_ERR_DATA, false, "%s is not a valid value for stage", stage); - return false; - } - - psString query = pxDataGet("magicdstool_getrunids.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, magic_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("magicdstool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - if (psArrayLength(output) > 1) { - psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found %ld for magic_id %" PRId64, - psArrayLength(output), magic_id); - return false; - } - psMetadata *row = output->data[0]; - - *cam_id = psMetadataLookupS64(NULL, row, "cam_id"); - switch (stageNum) { - case IPP_STAGE_RAW: - *stage_id = psMetadataLookupS64(NULL, row, "exp_id"); - break; - case IPP_STAGE_CHIP: - *stage_id = psMetadataLookupS64(NULL, row, "chip_id"); - break; - case IPP_STAGE_CAMERA: - *stage_id = *cam_id; - return true; - case IPP_STAGE_WARP: - *stage_id = psMetadataLookupS64(NULL, row, "warp_id"); - break; - case IPP_STAGE_DIFF: - *stage_id = psMetadataLookupS64(NULL, row, "diff_id"); - break; - } - - return true; -} - -static bool magicDSRunComplete(pxConfig *config, bool setmagicked) -{ - PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); // look for completed magicDSRuns psString query = pxDataGet("magicdstool_completed_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); } @@ -766,4 +929,8 @@ return true; } + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } for (long i = 0; i < psArrayLength(output); i++) { psMetadata *row = output->data[i]; @@ -771,20 +938,30 @@ psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); - // if requested, set stageRun.magicked + // if re_place, set stageRun.magicked + bool setmagicked = psMetadataLookupBool(NULL, row, "re_place"); if (setmagicked && !setRunMagicked(config, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64, magic_ds_id); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } // set magicDSRun.state to 'full' - if (!setmagicDSRunState(config, magic_ds_id, "full")) { + if (!setmagicDSRunState(config, magic_ds_id, NULL, NULL, "full")) { psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64, magic_ds_id); psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } } - + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -795,4 +972,10 @@ { PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", true); + if (!i_am_sure) { + psError(PS_ERR_UNKNOWN, true, "Reverting destreaked files must be done carefully. -i_am_sure is required."); + return false; + } psMetadata *where = psMetadataAlloc(); @@ -802,5 +985,5 @@ pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - psString query = psStringCopy("DELETE FROM magicDSFile USING magicDSFile, magicDSRun WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) AND magicDSFile.fault != 0"); + psString query = pxDataGet("magicdstool_revertdestreakedfile.sql"); if (psListLength(where->list)) { @@ -808,4 +991,7 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; } psFree(where); @@ -813,40 +999,89 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to revert"); - return false; - } + psFree(query); + return false; + } + psFree(query); return true; } - -static bool getskycellsMode(pxConfig *config) -{ - // required - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); +static bool clearstatefaultsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magicdstool_getskycells.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + // new state + PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false); + // old state (required) + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("magicdstool_clearstatefaults.sql"); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; } psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!new_state) { + if (!strcmp(state, "failed_revert")) { + new_state = "new"; + } else if (!strcmp(state, "failed_cleanup")) { + new_state = "goto_cleaned"; + } else { + psError(PS_ERR_UNKNOWN, true, "unexpected value for state: %s", state); + return false; + } + } + if (!p_psDBRunQueryF(config->dbh, query, new_state)) { + psError(PS_ERR_UNKNOWN, false, "failed to revert"); psFree(query); return false; } psFree(query); - + return true; +} + +static bool completedrevertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("magicdstool_completedrevert.sql"); + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + psString whereString = NULL; + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) { + psFree(whereString); + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + psFree(whereString); + psFree(query); psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -868,14 +1103,24 @@ return true; } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); + for (int i=0; idata[i]; + psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); + psString old_state = psMetadataLookupStr(NULL, row, "state"); + psString new_state; + if (!strcmp(old_state, "goto_censored")) { + new_state = "censored"; + } else if (!strcmp(old_state, "goto_restored")) { + new_state = "restored"; + } else { + psError(PXTOOLS_ERR_PROG, true, "unexpected state found: %s", old_state); psFree(output); return false; } - } - + char *query2 = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query2, new_state, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to set run magicDSRun.state to %s", new_state); + return false; + } + } psFree(output); @@ -883,63 +1128,30 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!( - (strncmp(state, "new", 4) == 0) - || (strncmp(state, "full", 5) == 0) - ) - ) { - psError(PS_ERR_UNKNOWN, false, - "invalid magicDSRun state: %s", state); - return false; - } - - char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for magic_id %" PRId64, magic_ds_id); - return false; - } - - return true; -} - -static bool toremoveMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); +static bool getskycellsMode(pxConfig *config) +{ + // required + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_toremove.sql"); + psString query = pxDataGet("magicdstool_getskycells.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); + whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause, "\n AND "); } psFree(where); - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -970,5 +1182,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -981,5 +1193,66 @@ return true; } -static bool torestoreMode(pxConfig *config) + +static bool validDSRunState(const char *state) +{ + if (!((strcmp(state, "new") == 0) || + (strcmp(state, "full") == 0) || + (strcmp(state, "drop") == 0) || + (strcmp(state, "failed_revert") == 0) || + (strcmp(state, "failed_cleanup") == 0) || + (strcmp(state, "restored") == 0) || + (strcmp(state, "censored") == 0) || + (strcmp(state, "cleaned") == 0) || + (strcmp(state, "goto_restored") == 0) || + (strcmp(state, "goto_censored") == 0) || + (strcmp(state, "goto_cleaned") == 0)) + ) { + return false; + } else { + return true; + } +} + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psString extraSetStr, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + if (!validDSRunState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid magicDSRun state: %s", state); + return false; + } + + psString query = NULL; + psStringAppend(&query, "UPDATE magicDSRun SET state = '%s' %s\n", state, extraSetStr ? extraSetStr : ""); + if (magic_ds_id) { + psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id); + } else if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; + } + + if (!strcmp(state, "goto_cleaned")) { + // Don't set state back to goto_cleaned if it is already cleaned + psStringAppend(&query, " AND (magicDSRun.state != 'cleaned')"); + + // don't clean up magicDSRun's where stage is camera + psStringAppend(&query, " AND (magicDSRun.stage != 'camera')"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for magic_id %" PRId64, magic_ds_id); + return false; + } + + return true; +} + +static bool toremoveMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -994,7 +1267,7 @@ // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_torestore.sql"); + psString query = pxDataGet("magicdstool_toremove.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1043,5 +1316,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -1055,5 +1328,4 @@ } - static bool torevertMode(pxConfig *config) { @@ -1075,5 +1347,5 @@ psString query = pxDataGet(sql_file); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement from %s", sql_file); psFree(sql_file); return false; @@ -1137,2 +1409,76 @@ +static bool tocleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magicdstool_tocleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/magicdstool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/magicdstool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/magicdstool.h (revision 27840) @@ -26,13 +26,16 @@ MAGICDSTOOL_MODE_NONE = 0x0, MAGICDSTOOL_MODE_DEFINEBYQUERY, - MAGICDSTOOL_MODE_DEFINERUN, + MAGICDSTOOL_MODE_DEFINECOPY, MAGICDSTOOL_MODE_UPDATERUN, MAGICDSTOOL_MODE_TODESTREAK, MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, + MAGICDSTOOL_MODE_ADVANCERUN, MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, + MAGICDSTOOL_MODE_CLEARSTATEFAULTS, MAGICDSTOOL_MODE_GETSKYCELLS, MAGICDSTOOL_MODE_TOREMOVE, - MAGICDSTOOL_MODE_TORESTORE, MAGICDSTOOL_MODE_TOREVERT, + MAGICDSTOOL_MODE_COMPLETEDREVERT, + MAGICDSTOOL_MODE_TOCLEANUP, } MAGICDStoolMode; Index: branches/simtest_nebulous_branches/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/magicdstoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/magicdstoolConfig.c (revision 27840) @@ -40,5 +40,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -52,6 +52,8 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-replace", 0, "replace input files with the destreaked versions", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL); @@ -64,24 +66,44 @@ psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-streaks_max", 0, "maximum number of streaks", 0); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one arleady exists", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be queued", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one already exists", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be queued", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - // -definerun - psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magic_id (required)", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for run (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-replace", 0, "use the simple output format", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -definecopy + psMetadata *definecopyArgs = psMetadataAlloc(); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-stage_label", PS_META_DUPLICATE_OK, "search by stageRun.label", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-magic_label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id", 0); + psMetadataAddS64(definecopyArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic_id", 0); + psMetadataAddS32(definecopyArgs, PS_LIST_TAIL, "-streaks_max", 0, "maximum number of streaks", 0); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one already exists", false); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be queued", false); + psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(definecopyArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "set data_group", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "data_group (LIKE) comparision", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-noreplace", 0, "only update runs with replace not set", false); // -addinputskyfile @@ -93,8 +115,8 @@ // -todestreak psMetadata *todestreakArgs = psMetadataAlloc(); + psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage (required)", NULL); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL); psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -107,4 +129,7 @@ psMetadataAddStr(adddestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL); psMetadataAddBool(adddestreakedfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false); + psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-streak_frac", 0, "set fraction of pixels masked by streaks", 0); + psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-nondiff_frac", 0, "set fraction of pixels masked because nondiffed", 0); + psMetadataAddF32(adddestreakedfileArgs, PS_LIST_TAIL, "-run_time", 0, "set the streaksremove run time for component ", 0); psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -114,9 +139,18 @@ psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL); psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertdestreakedfileArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirm that you know what you are doing", false); psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + + // -clearstatefaults + psMetadata *clearstatefaultsArgs = psMetadataAlloc(); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); + psMetadataAddS16(clearstatefaultsArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); // -getskycells psMetadata *getskycellsArgs = psMetadataAlloc(); - psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID (required)", 0); + psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID", 0); psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); @@ -130,12 +164,4 @@ psMetadataAddU64(toremoveArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(toremoveArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -torestore - psMetadata *torestoreArgs = psMetadataAlloc(); - psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); - psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); - psMetadataAddStr(torestoreArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddU64(torestoreArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(torestoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -torevert @@ -148,4 +174,24 @@ psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -completedrevert + psMetadata *completedrevertArgs = psMetadataAlloc(); + psMetadataAddS64(completedrevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(completedrevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(completedrevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -advancerun + psMetadata *advancerunArgs = psMetadataAlloc(); + psMetadataAddS64(advancerunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(advancerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanup + psMetadata *tocleanupArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-stage", 0, "define output directory", NULL); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -155,6 +201,6 @@ PXOPT_ADD_MODE("-definebyquery", "create magic de-streak runs from magic runs", MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "create one magic de-streak run for one magic run", - MAGICDSTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-definecopy", "create magic de-streak runs from magic runs for different path", + MAGICDSTOOL_MODE_DEFINECOPY, definecopyArgs); PXOPT_ADD_MODE("-updaterun", "update state of magic de-streak run", MAGICDSTOOL_MODE_UPDATERUN, updaterunArgs); @@ -163,14 +209,22 @@ PXOPT_ADD_MODE("-adddestreakedfile", "add a de-streaked file", MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileArgs); + PXOPT_ADD_MODE("-advancerun", "change state for runs that have finished destrreaking", + MAGICDSTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-revertdestreakedfile", " revert a faulted de-streaked file", MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, revertdestreakedfileArgs); + PXOPT_ADD_MODE("-clearstatefaults", " clear magicDSRuns in fault state", + MAGICDSTOOL_MODE_CLEARSTATEFAULTS, clearstatefaultsArgs); PXOPT_ADD_MODE("-getskycells", "get skycell files ", MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsArgs); PXOPT_ADD_MODE("-toremove", "backup images pending removal", MAGICDSTOOL_MODE_TOREMOVE, toremoveArgs); - PXOPT_ADD_MODE("-torestore", "images pending restore of exicsed streak pixels", - MAGICDSTOOL_MODE_TORESTORE, torestoreArgs); - PXOPT_ADD_MODE("-torevert", "faulted images to revert", + PXOPT_ADD_MODE("-torevert", "images to restore or revert", MAGICDSTOOL_MODE_TOREVERT, torevertArgs); + + PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting", + MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs); + PXOPT_ADD_MODE("-tocleanup", "destreak runs to clean up", + MAGICDSTOOL_MODE_TOCLEANUP, tocleanupArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/simtest_nebulous_branches/ippTools/src/magictool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/magictool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/magictool.c (revision 27840) @@ -114,7 +114,9 @@ // Required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); // Optional - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-note", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -133,5 +135,5 @@ psString query = pxDataGet("magictool_definebyquery_select.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -149,6 +151,8 @@ // what if no skycells for the diff run completed? - if (!rerun) { - psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE"); + psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE"); + psString rerunWhereStr = NULL; + if (rerun) { + psStringAppend(&rerunWhereStr, "\n WHERE magicRun.label = '%s'", label); } @@ -174,9 +178,13 @@ queryWhereStr = psStringCopy(""); } - - if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) { + if (!rerunWhereStr) { + rerunWhereStr = psStringCopy(""); + } + + if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, rerunWhereStr, queryWhereStr)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(diffWhereStr); psFree(queryWhereStr); + psFree(rerunWhereStr); psFree(query); return false; @@ -184,4 +192,5 @@ psFree(diffWhereStr); psFree(queryWhereStr); + psFree(rerunWhereStr); psFree(query); } @@ -236,6 +245,17 @@ // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label, - dvodb, registered, 0); + magicRunRow *run = magicRunRowAlloc(0, + exp_id, + diff_id, + inverse, + "new", // state + workdir, + "dirty", // workdir_state + label, + data_group ? data_group : label, + dvodb, + registered, + 0, // fault + note); if (!run) { psAbort("failed to alloc magicRun object"); @@ -331,7 +351,9 @@ "dirty", // workdir_state label, + NULL, // data_group dvodb, registered, - 0 + 0, // fault + NULL ); @@ -412,5 +434,5 @@ psString query = pxDataGet("magictool_inputskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -485,5 +507,5 @@ psString query = pxDataGet("magictool_totree.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -558,5 +580,5 @@ if (fault > 0) { - char *query = "UPDATE magicRun SET fault = %d, state = 'full' WHERE magic_id = %" PRId64; + char *query = "UPDATE magicRun SET fault = %d WHERE magic_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, fault, magic_id)) { psError(PS_ERR_UNKNOWN, false, @@ -597,4 +619,7 @@ return false; } + psS32 numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("magictool", PS_LOG_INFO, "Reverted %d magic runs", numUpdated); + return true; } @@ -620,5 +645,5 @@ psString query = pxDataGet("magictool_inputs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -790,5 +815,5 @@ psString query = pxDataGet("magictool_toprocess_inputs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -852,12 +877,21 @@ // look for tree nodes that need to be processed + + // first find incomplete magicRuns query = pxDataGet("magictool_toprocess_runs.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // we limit the query even though it is cheap (only magic_id is selected) + + // XXX: if the first 1000 unfinished magicRuns have no ready nodes + // that haven't faulted, later runs won't get returned even though + // they have work to do. When we used a limit of 100 we actually ran + // into this problem. Since we're using labels a limit of 1000 will + // probably be ok. { - psString limitString = psDBGenerateLimitSQL( 100 ); + psString limitString = psDBGenerateLimitSQL( 1000 ); psStringAppend(&query, " %s", limitString); psFree(limitString); @@ -895,5 +929,5 @@ query = pxDataGet("magictool_toprocess_tree.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -908,5 +942,5 @@ psAbort("failed to lookup value for magic_id column"); } - + whereString = NULL; psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id); @@ -1026,10 +1060,10 @@ PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "magicNodeResult.fault", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); psString query = pxDataGet("magictool_revertnode.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1052,5 +1086,5 @@ psS32 numUpdated = psDBAffectedRows(config->dbh); - psLogMsg("magictool", PS_LOG_INFO, "Updated %d magic nodes", numUpdated); + psLogMsg("magictool", PS_LOG_INFO, "Reverted %d magic nodes", numUpdated); return true; @@ -1068,5 +1102,5 @@ psString query = pxDataGet("magictool_tomask.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1156,5 +1190,5 @@ psString query = pxDataGet("magictool_addmask.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1263,5 +1297,5 @@ psString query = pxDataGet("magictool_mask.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1393,46 +1427,25 @@ } -static bool censorStage(pxConfig *config, psString stage, psString whereClause) -{ - psString queryFile = NULL; - psStringAppend(&queryFile, "magictool_censor_%s.sql", stage); - psString query = pxDataGet(queryFile); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); - psFree(queryFile); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(queryFile); - - psStringAppend(&query, " WHERE %s", whereClause); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - return true; -} - static bool censorrunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - psError(PS_ERR_PROGRAMMING, true, "-censorrun mode not ready yet"); - return false; - psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + if (!magic_id) { + if (!exp_id && !label) { + psError(PS_ERR_UNKNOWN, true, "either -magic_id or exp_id and label is required"); + return false; + } + } // at least one of these required PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); if (!psListLength(where->list)) { @@ -1470,22 +1483,22 @@ // Now queue any destreaked files to be re-verted - // note: on failure censorStage issues the rollback - if (!censorStage(config, "raw", whereClause)) { + // note: on failure pxmagicRestoreStage issues the rollback + if (!pxmagicRestoreStage(config, "raw", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "chip", whereClause)) { + if (!pxmagicRestoreStage(config, "chip", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "camera", whereClause)) { + if (!pxmagicRestoreStage(config, "camera", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "warp", whereClause)) { + if (!pxmagicRestoreStage(config, "warp", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "diff", whereClause)) { + if (!pxmagicRestoreStage(config, "diff", whereClause, "goto_censored")) { psFree(whereClause); return false; Index: branches/simtest_nebulous_branches/ippTools/src/magictoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/magictoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/magictoolConfig.c (revision 27840) @@ -40,5 +40,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -50,7 +50,9 @@ psMetadata *definebyqueryArgs = psMetadataAlloc(); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label (required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-note", 0, "define note", NULL); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search diff_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); @@ -135,5 +137,5 @@ psMetadataAddS64(revertnodeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-node", 0, "search by node name", NULL); - psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-label",0, "search by label", NULL); + psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); psMetadataAddS16(revertnodeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -165,4 +167,5 @@ psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(censorrunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psFree(now); Index: branches/simtest_nebulous_branches/ippTools/src/pstamptool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pstamptool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pstamptool.c (revision 27840) @@ -44,7 +44,12 @@ static bool pendingjobMode(pxConfig *config); static bool updatejobMode(pxConfig *config); +static bool revertjobMode(pxConfig *config); static bool addprojectMode(pxConfig *config); static bool projectMode(pxConfig *config); static bool modprojectMode(pxConfig *config); +static bool getdependentMode(pxConfig *config); +static bool pendingdependentMode(pxConfig *config); +static bool updatedependentMode(pxConfig *config); +static bool revertdependentMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -80,7 +85,12 @@ MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode); MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode); + MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode); MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode); MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); MODECASE(PSTAMPTOOL_MODE_PROJECT, projectMode); + MODECASE(PSTAMPTOOL_MODE_GETDEPENDENT, getdependentMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentMode); + MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode); default: psAbort("invalid option (this should not happen)"); @@ -108,8 +118,10 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - PXOPT_LOOKUP_STR(outProduct, config->args, "-out_product", true, false); - PXOPT_LOOKUP_STR(lastFileset, config->args, "-last_fileset", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-set_uri", true, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-set_out_product", true, false); + PXOPT_LOOKUP_STR(lastFileset, config->args, "-set_last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval",false, false); if (!pstampDataStoreInsert(config->dbh, @@ -117,6 +129,9 @@ state, lastFileset, + NULL, // timestamp + label, // label outProduct, - uri + uri, + pollInterval )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -134,9 +149,10 @@ PXOPT_COPY_S64(config->args, where, "-ds_id", "ds_id", "=="); + PXOPT_LOOKUP_BOOL(ready, config->args, "-ready", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); psString query = pxDataGet("pstamptool_datastore.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -148,4 +164,7 @@ } psFree(where); + if (ready) { + psStringAppend(&query, " %s", "\nAND TIMESTAMPDIFF(SECOND, timestamp, now()) > pollInterval"); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -181,25 +200,33 @@ PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", true, false); - PXOPT_LOOKUP_STR(lastFileset, config->args, "-last_fileset", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); - - if (!state && !lastFileset) { - psError(PS_ERR_UNKNOWN, true, "at least one of -last_fileset or -state is required"); - return false; - } - - char *query = psStringCopy ("UPDATE pstampDataStore SET"); - bool needComma = false; - + PXOPT_LOOKUP_STR(lastFileset, config->args, "-set_last_fileset", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval", false, false); + PXOPT_LOOKUP_BOOL(update_timestamp, config->args, "-update_timestamp", false); + + if (!state && !lastFileset && !pollInterval && !update_timestamp && !label) { + psError(PS_ERR_UNKNOWN, true, "at least one of -last_fileset or -set_state is required"); + return false; + } + + char *query = psStringCopy ("UPDATE pstampDataStore SET timestamp = CURRENT_TIMESTAMP() "); + if (lastFileset) { - psStringAppend(&query, " lastFileset = '%s'", lastFileset); - needComma = true; + psStringAppend(&query, " , lastFileset = '%s'", lastFileset); } if (state) { - psStringAppend(&query, "%s state = '%s'", needComma ? "," : " ", state); - needComma = true; // be ready in case we add another field - } - + psStringAppend(&query, ", state = '%s'", state); + } + + if (label) { + psStringAppend(&query, ", label = '%s'", label); + } + + if (pollInterval) { + psStringAppend(&query, ", pollInterval = %d", pollInterval); + } + psStringAppend(&query, " WHERE ds_id = %" PRId64, ds_id); @@ -212,5 +239,5 @@ psU64 affected = psDBAffectedRows(config->dbh); if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); return false; @@ -224,21 +251,21 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - // PXOPT_LOOKUP_STR(outFileset, config->args, "-out_fileset", true, false); - PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); - - char *query ="INSERT INTO pstampRequest" - " (state, uri, ds_id, fault)" - " VALUES( 'new', '%s', %" PRId64 ", 0 )"; - if (!p_psDBRunQueryF(config->dbh, query, uri, ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psU64 affected = psDBAffectedRows(config->dbh); - if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, - "should have affected one row but %" PRIu64 " rows were modified", - affected); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); + + if (!pstampRequestInsert(config->dbh, + 0, // req_id + ds_id, + "new", //state + name, + NULL, // reqType + label, + NULL, // outProduct + uri, + 0 // fault + )) { + psError(PS_ERR_UNKNOWN, false, "failed to insert request"); return false; } @@ -257,4 +284,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -263,5 +291,5 @@ psString query = pxDataGet("pstamptool_pendingreq.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -315,11 +343,24 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = NULL; - psStringAppend(&query, "SELECT * from pstampRequest WHERE req_id = %" PRId64, req_id); - + if (!psListLength(where->list)) { + psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); + return false; + } + + psString query = psStringCopy("SELECT * from pstampRequest"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + // treat limit == 0 as "no limit" if (limit) { @@ -342,7 +383,10 @@ } if (!psArrayLength(output)) { - psTrace("pstamptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; + psTrace("pstamptool", PS_LOG_INFO, "request not found"); + // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for + psError(PXTOOLS_ERR_CONFIG, true, "request not found"); + psFree(output); + // we return false so that the caller can determine that a request does not exist + return false; } @@ -363,11 +407,18 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = - "SELECT * FROM pstampRequest WHERE state = 'run' AND " - "(SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id " - " AND pstampJob.state != 'stop') = 0" ; + psString query = pxDataGet("pstamptool_completedreq.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" @@ -415,7 +466,7 @@ PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(outProduct, config->args, "-outProduct", false, false); - PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); PXOPT_LOOKUP_STR(reqType, config->args, "-reqType", false, false); @@ -433,5 +484,5 @@ } if (fault) { - psStringAppend(&query, "%c fault = %s", c, fault); + psStringAppend(&query, "%c fault = %d", c, fault); c = ','; } @@ -464,5 +515,5 @@ // note zero is not an error if (affected > 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); return false; @@ -476,17 +527,41 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); - - // printf("Revert request %" PRId64 "\n", req_id); - - if (!p_psDBRunQueryF(config->dbh, "DELETE FROM pstampJob where req_id = %" PRId64, req_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!p_psDBRunQueryF(config->dbh, - "UPDATE pstampRequest set state ='new', name=NULL, reqType=NULL, fault=0 where req_id = %" PRId64, req_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + + // delete any jobs that were queued by requests that didn't complete parsing (pstampRequest.state = 'new' + // If state = 'run' was supplied this will be a no-op + psString query = pxDataGet("pstamptool_revertreq_deletejobs.sql"); + psStringAppend(&query, " AND %s", whereClause); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + // clear fault for requests + query = pxDataGet("pstamptool_revertreq.sql"); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); return true; @@ -506,29 +581,33 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); // unless the job is being inserted with stop state require outputBase if (strcmp(stateString, "stop") && !outputBase) { - psError(PS_ERR_UNKNOWN, true, "-outputBase is required"); - return false; + psError(PS_ERR_UNKNOWN, true, "-outputBase is required"); + return false; } // default value for job_type is defined in pstamptoolConfig.c - if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query")) { - stampJob = false; + if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query") || !strcmp(job_type, "none")) { + stampJob = false; } else if (!strcmp(job_type, "stamp")) { - stampJob = true; + stampJob = true; } else { - psError(PS_ERR_UNKNOWN, false, "unknown value for -job_type: %s", job_type); - return false; - } - if (!pstampJobInsert(config->dbh, + psError(PS_ERR_UNKNOWN, false, "unknown value for -job_type: %s", job_type); + return false; + } + if (!pstampJobInsert(config->dbh, 0, // job_id req_id, - rownum, - stateString, - job_type, - fault, - exp_id, - outputBase + rownum, + stateString, + job_type, + fault, + exp_id, + outputBase, + options, + dep_id )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -538,5 +617,5 @@ psU64 affected = psDBAffectedRows(config->dbh); if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); @@ -554,34 +633,30 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", false, false); - PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = NULL; - - if (!req_id && !job_id) { - fprintf(stderr, "either req_id or job_id must be specified\n"); + if (!psListLength(where->list)) { + fprintf(stderr, "search arguments are required\n"); exit (1); } - if (req_id) { - psStringAppend(&query, - "SELECT" - " pstampJob.*, pstampRequest.name, pstampRequest.outProduct" - " FROM pstampJob" - " JOIN pstampRequest USING(req_id)" - " WHERE req_id = %" PRId64, req_id - ); - } else { - psStringAppend(&query, - "SELECT" - " pstampJob.*, pstampRequest.name, pstampRequest.outProduct" - " FROM pstampJob" - " JOIN pstampRequest USING(req_id)" - " WHERE job_id = %" PRId64, job_id - ); - } + psString query = pxDataGet("pstamptool_listjob.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" @@ -630,4 +705,6 @@ PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -635,5 +712,5 @@ psString query = pxDataGet("pstamptool_pendingjob.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -641,5 +718,5 @@ // use psDBGenerateWhereSQL because the SQL yields an intermediate table if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -688,7 +765,18 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false); + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); + + if (!job_id && !dep_id) { + psError(PS_ERR_UNKNOWN, true, "at least -job_id or -dep_id is required"); + return false; + } PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); psString faultStr = NULL; @@ -696,12 +784,15 @@ faultStr = psStringCopy(""); } else { - psStringAppend(&faultStr, ", fault = '%s'", fault); - } - - char *query ="UPDATE pstampJob" - " SET state = '%s' %s" - " WHERE job_id = %" PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, faultStr, job_id)) { + psStringAppend(&faultStr, "\n, pstampJob.fault = %d", fault); + } + + psString query = pxDataGet("pstamptool_updatejob.sql"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, state, faultStr)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -709,9 +800,51 @@ } psFree(faultStr); + psFree(query); psU64 affected = psDBAffectedRows(config->dbh); - if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" - PRIu64 " rows were modified", affected); + psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampJobs", affected); + + return true; +} + +static bool revertjobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "pstampJob.fault", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id_min", "req_id", ">="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + // By default only revert faults < 10 which are our "ipp exit codes" + // codes larger than that are the pstamp request interface. + // Don't fault those unless -fault waa provided + psString faultClause = ""; + if (!fault) { + faultClause = " \nAND (pstampJob.fault < 10)"; + } + + psString query = pxDataGet("pstamptool_revertjob.sql"); + if (!psListLength(where->list) && !all) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters or -all are required"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, faultClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); return false; } @@ -759,5 +892,5 @@ psString query = pxDataGet("pstamptool_project.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -805,7 +938,7 @@ char *query = psStringCopy ("UPDATE pstampProject SET"); - + psStringAppend(&query, " state = '%s'", state); - + psStringAppend(&query, " WHERE proj_id = %" PRId64, proj_id); @@ -818,5 +951,5 @@ psU64 affected = psDBAffectedRows(config->dbh); if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); return false; @@ -826,2 +959,257 @@ } +static bool getdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); + PXOPT_LOOKUP_STR(imagedb, config->args, "-imagedb", true, false); + PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", true, false); + PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false); + PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); + + // start a transaction eraly so it will contain any row level locks + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString query = pxDataGet("pstamptool_getdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (!no_create) { + // This will lock the row until the transaction is committed + psStringAppend(&query, " AND %s FOR UPDATE", whereClause); + } else { + psStringAppend(&query, " AND %s", whereClause); + } + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (psArrayLength(output)) { + psMetadata *dep = output->data[0]; + psS64 dep_id = psMetadataLookupS64(NULL, dep, "dep_id"); + if (!dep_id) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + printf("%" PRId64 "\n", dep_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return true; + } + if (no_create) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + // no existing dependent that matches, insert one + // Since we have multiple processes running jobs we have a + // race condition here so that's why we need to lock the table + + if (!pstampDependentInsert( + config->dbh, + 0, // dep_id + "new", // state + stage, + stage_id, + component, + imagedb, + rlabel, + need_magic, + 0 // fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to insert pstampDependent"); + return false; + } + + // if we try and get this after commit zero is returned + psS64 dep_id = psDBLastInsertID(config->dbh); + if (!dep_id) { + psError(PS_ERR_UNKNOWN, false, "psDBLastInsertID returned NULL"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + printf("%" PRId64 "\n", dep_id); + + return true; +} + +static bool pendingdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-imagedb", "imagedb", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-rlabel", "rlabel", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_pendingdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampDependent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool updatedependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + if (!fault && !state) { + psError(PS_ERR_UNKNOWN, true, "at least one of -set_state or fault is required"); + return false; + } + psString query = psStringCopy("UPDATE pstampDependent SET"); + bool needComma = false; + if (state) { + psStringAppend(&query, " state = '%s'", state); + needComma = true; + } + if (fault) { + psStringAppend(&query, "%s fault = %d", needComma ? ", " : "", fault); + needComma = true; + } + psStringAppend(&query, " WHERE dep_id = %" PRId64, dep_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} +static bool revertdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-fault", "pstampDependent.fault", "=="); + PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = pxDataGet("pstamptool_revertdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/pstamptool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pstamptool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pstamptool.h (revision 27840) @@ -39,7 +39,12 @@ PSTAMPTOOL_MODE_JOBRESULT, PSTAMPTOOL_MODE_UPDATEJOB, + PSTAMPTOOL_MODE_REVERTJOB, PSTAMPTOOL_MODE_ADDPROJECT, PSTAMPTOOL_MODE_MODPROJECT, PSTAMPTOOL_MODE_PROJECT, + PSTAMPTOOL_MODE_GETDEPENDENT, + PSTAMPTOOL_MODE_PENDINGDEPENDENT, + PSTAMPTOOL_MODE_UPDATEDEPENDENT, + PSTAMPTOOL_MODE_REVERTDEPENDENT, } pstamptoolMode; Index: branches/simtest_nebulous_branches/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pstamptoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pstamptoolConfig.c (revision 27840) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psError(psErrorCodeLast(), false, "Can't find site configuration!\n"); psFree(config); return NULL; @@ -45,29 +45,38 @@ // -adddatastore psMetadata *adddatastoreArgs = psMetadataAlloc(); - psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-uri", 0, "define storage uri (required)", NULL); - psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-out_product", 0, "define output product name (required)", NULL); - psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-last_fileset", 0, "define last fileset seen", NULL); - psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-state", 0, "define datastore state", "enabled"); - + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_uri", 0, "define storage uri (required)", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_out_product", 0, "define output product name (required)", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_last_fileset", 0, "define last fileset seen", NULL); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_state", 0, "define datastore state", "enabled"); + psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_label", 0, "define datastore label", NULL); + psMetadataAddS32(adddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 60); + // -datastore psMetadata *datastoreArgs = psMetadataAlloc(); - psMetadataAddS64(datastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddS64(datastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddBool(datastoreArgs, PS_LIST_TAIL, "-ready", 0, "list data stores ready to be polled", false); psMetadataAddBool(datastoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -moddatastore psMetadata *moddatastoreArgs = psMetadataAlloc(); - psMetadataAddS64(moddatastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); - psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-last_fileset", 0, "set last_fileset seen", NULL); - psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddS64(moddatastoreArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_last_fileset", 0, "set last_fileset seen", NULL); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddS32(moddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 0); + psMetadataAddBool(moddatastoreArgs, PS_LIST_TAIL, "-update_timestamp", 0, "update the timestamp", false); + psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); // -addreq psMetadata *addreqArgs = psMetadataAlloc(); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); - psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); - // psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-out_fileset", 0, "define request output_fileset", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0, "define request name", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0, "define request label", NULL); // -pendingreq psMetadata *pendingreqArgs = psMetadataAlloc(); - psMetadataAddS64(pendingreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddS64(pendingreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddStr(pendingreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); psMetadataAddU64(pendingreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -75,10 +84,13 @@ // -listreq psMetadata *listreqArgs = psMetadataAlloc(); - psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id (required)", 0); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "list by req_id", 0); + psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0, "list by name", NULL); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0, "req_id to not list", 0); psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -completedreq psMetadata *completedreqArgs = psMetadataAlloc(); + psMetadataAddStr(completedreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddU64(completedreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(completedreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -86,37 +98,44 @@ // -updatereq psMetadata *updatereqArgs = psMetadataAlloc(); - psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); + psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); // -revertreq psMetadata *revertreqArgs = psMetadataAlloc(); - psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to revert", 0); + psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS16(revertreqArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); + psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); // -addjob psMetadata *addjobArgs = psMetadataAlloc(); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "new state", "run"); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "exposure id", 0); - psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "new result", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "define state", "run"); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure id", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); + psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); // -listjob psMetadata *listjobArgs = psMetadataAlloc(); - psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); - psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); - psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); + psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); + psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -pendingjob psMetadata *pendingjobArgs = psMetadataAlloc(); - psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); - psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); + psMetadataAddS64(pendingjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); + psMetadataAddStr(pendingjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddU64(pendingjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -124,7 +143,51 @@ // -updatejob psMetadata *updatejobArgs = psMetadataAlloc(); - psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); + psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id for which to change state", 0); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); + psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", 0); + + // -revertjob + psMetadata *revertjobArgs = psMetadataAlloc(); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id_min", 0, "minimum req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id to revert", 0); + psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddStr(revertjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); + psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted jobs", false); + psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -getdependent + psMetadata *getdependentArgs = psMetadataAlloc(); + psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent (required)", 0); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for depenent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent ", NULL); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-need_magic", 0, "define need_magic", false); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no matching dependent do not create one", false); + + // -updatedependent + psMetadata *updatedependentArgs = psMetadataAlloc(); + psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent (required)", 0); + psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); + psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-fault", 0, "new value for fault", 0); + + // -revertdependent + psMetadata *revertdependentArgs = psMetadataAlloc(); + psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent", 0); + psMetadataAddStr(revertdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for dependent ", NULL); + psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault", 0, "define fault", 0); + + // -pendingdependent + psMetadata *pendingdependentArgs = psMetadataAlloc(); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent", 0); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-component", 0, "define component for dependent", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label for pstampRequest", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent", NULL); + psMetadataAddU64(pendingdependentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingdependentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -addproject @@ -140,5 +203,5 @@ // -modproject psMetadata *modprojectArgs = psMetadataAlloc(); - psMetadataAddS64(modprojectArgs, PS_LIST_TAIL, "-proj_id", 0, "define project ID to modify (required)", 0); + psMetadataAddS64(modprojectArgs, PS_LIST_TAIL, "-proj_id", 0, "define project ID to modify (required)", 0); psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-imagedb", 0, "define name of database for project", NULL); psMetadataAddStr(modprojectArgs, PS_LIST_TAIL, "-state", 0, "define state for project (enabled, disabled)", NULL); @@ -150,5 +213,5 @@ // -project psMetadata *projectArgs = psMetadataAlloc(); - psMetadataAddStr(projectArgs, PS_LIST_TAIL, "-name", 0, "define project name to list (required)", NULL); + psMetadataAddStr(projectArgs, PS_LIST_TAIL, "-name", 0, "define project name to list (required)", NULL); psMetadataAddBool(projectArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -158,5 +221,5 @@ PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); - PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); + PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); PXOPT_ADD_MODE("-listreq", "", PSTAMPTOOL_MODE_LISTREQ, listreqArgs); PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); @@ -166,9 +229,15 @@ PXOPT_ADD_MODE("-listjob", "", PSTAMPTOOL_MODE_LISTJOB, listjobArgs); PXOPT_ADD_MODE("-pendingjob", "", PSTAMPTOOL_MODE_PENDINGJOB, pendingjobArgs); - PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + PXOPT_ADD_MODE("-revertjob", "", PSTAMPTOOL_MODE_REVERTJOB, revertjobArgs); PXOPT_ADD_MODE("-adddatastore", "", PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreArgs); PXOPT_ADD_MODE("-datastore", "", PSTAMPTOOL_MODE_DATASTORE, datastoreArgs); PXOPT_ADD_MODE("-moddatastore", "", PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreArgs); + + PXOPT_ADD_MODE("-getdependent", "", PSTAMPTOOL_MODE_GETDEPENDENT, getdependentArgs); + PXOPT_ADD_MODE("-updatedependent", "", PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentArgs); + PXOPT_ADD_MODE("-pendingdependent","", PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentArgs); + PXOPT_ADD_MODE("-revertdependent","", PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentArgs); PXOPT_ADD_MODE("-addproject", "", PSTAMPTOOL_MODE_ADDPROJECT, addprojectArgs); Index: branches/simtest_nebulous_branches/ippTools/src/pubtool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pubtool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pubtool.c (revision 27840) @@ -32,4 +32,5 @@ static bool defineclientMode(pxConfig *config); +static bool updateclientMode(pxConfig *config); static bool definerunMode(pxConfig *config); static bool pendingMode(pxConfig *config); @@ -57,4 +58,5 @@ switch (config->mode) { MODECASE(PUBTOOL_MODE_DEFINECLIENT, defineclientMode); + MODECASE(PUBTOOL_MODE_UPDATECLIENT, updateclientMode); MODECASE(PUBTOOL_MODE_DEFINERUN, definerunMode); MODECASE(PUBTOOL_MODE_PENDING, pendingMode); @@ -93,9 +95,52 @@ // optional PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); - - if (!publishClientInsert(config->dbh, 0, product, stage, workdir, comment)) { - psError(PS_ERR_UNKNOWN, false, "Database error"); - return false; - } + PXOPT_LOOKUP_BOOL(unmagicked, config->args, "-unmagicked", false); + + if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + return true; +} + +static bool updateclientMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-product", "product", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); + + PXOPT_LOOKUP_BOOL(active, config->args, "-active", false); + PXOPT_LOOKUP_BOOL(inactive, config->args, "-inactive", false); + + if ((!active && !inactive) || (active && inactive)) { + psError(PS_ERR_UNKNOWN, false, "Must specify one, and only one, of -active and -inactive."); + psFree(where); + return false; + } + + psString query = NULL; // Query to run + psStringAppend(&query, "UPDATE publishClient SET active = %d", active ? 1 : 0); + + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\n WHERE %s", clause); + psFree(clause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + return false; + } + psFree(query); + + long numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("pubtool", PS_LOG_INFO, "%ld rows updated.", numUpdated); return true; @@ -112,11 +157,9 @@ // optional PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); // define using newExp label psString query = pxDataGet("pubtool_definerun.sql"); // Query to run if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -167,4 +210,5 @@ psS64 client = psMetadataLookupS64(NULL, row, "client_id"); // Client identifier psS64 stage = psMetadataLookupS64(NULL, row, "stage_id"); // Stage identifier + char *label = psMetadataLookupStr(NULL, row, "src_label"); // label from correct source if (!publishRunInsert(config->dbh, 0, client, stage, label, "new")) { @@ -205,5 +249,5 @@ psString query = pxDataGet("pubtool_pending.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -259,7 +303,9 @@ // required PXOPT_LOOKUP_S64(pub_id, config->args, "-pub_id", true, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); // optional + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); @@ -269,5 +315,5 @@ } - if (!publishDoneInsert(config->dbh, pub_id, path_base, fault)) { + if (!publishDoneInsert(config->dbh, pub_id, path_base, hostname, dtime_script, fault)) { psError(PS_ERR_UNKNOWN, false, "Unable to add file"); if (!psDBRollback(config->dbh)) { @@ -309,9 +355,9 @@ PXOPT_COPY_STR(config->args, where, "-client_id", "publishClient.client_id", "=="); PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-label", "publishRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-label", "publishRun.label", "=="); psString query = pxDataGet("pubtool_revert.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -332,5 +378,7 @@ psFree(query); - return true; -} - + psLogMsg("pubtool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + + return true; +} + Index: branches/simtest_nebulous_branches/ippTools/src/pubtool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pubtool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pubtool.h (revision 27840) @@ -26,4 +26,5 @@ PUBTOOL_MODE_NONE = 0x0, PUBTOOL_MODE_DEFINECLIENT, + PUBTOOL_MODE_UPDATECLIENT, PUBTOOL_MODE_DEFINERUN, PUBTOOL_MODE_PENDING, Index: branches/simtest_nebulous_branches/ippTools/src/pubtoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pubtoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pubtoolConfig.c (revision 27840) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!"); + psError(psErrorCodeLast(), false, "Can't find site configuration!"); psFree(config); return NULL; @@ -49,9 +49,19 @@ psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + psMetadataAddBool(defineclientArgs, PS_LIST_TAIL, "-unmagicked", 0, "allow unmagicked data?", false); + + // -updateclient + psMetadata *updateclientArgs = psMetadataAlloc(); + psMetadataAddS64(updateclientArgs, PS_LIST_TAIL, "-client_id", 0, "search by client_id", 0); + psMetadataAddStr(updateclientArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); + psMetadataAddStr(updateclientArgs, PS_LIST_TAIL, "-product", 0, "search by product", NULL); + psMetadataAddStr(updateclientArgs, PS_LIST_TAIL, "-comment", 0, "search by comment (LIKE)", NULL); + psMetadataAddBool(updateclientArgs, PS_LIST_TAIL, "-active", 0, "set to active state", NULL); + psMetadataAddBool(updateclientArgs, PS_LIST_TAIL, "-inactive", 0, "set to inactive state", NULL); // -definerun psMetadata *definerunArgs = psMetadataAlloc(); psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-client_id", 0, "search by client_id", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "set and search by label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "set and search by label", NULL); // -pending @@ -68,4 +78,6 @@ psMetadataAddS64(addArgs, PS_LIST_TAIL, "-pub_id", 0, "define pub_id (required)", 0); psMetadataAddStr(addArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL); + psMetadataAddStr(addArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddF32(addArgs, PS_LIST_TAIL, "-dtime_script", 0, "define time for script", NAN); psMetadataAddS32(addArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); @@ -76,5 +88,5 @@ psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", NULL); psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); - psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on label", NULL); @@ -83,4 +95,5 @@ PXOPT_ADD_MODE("-defineclient", "", PUBTOOL_MODE_DEFINECLIENT, defineclientArgs); + PXOPT_ADD_MODE("-updateclient", "", PUBTOOL_MODE_UPDATECLIENT, updateclientArgs); PXOPT_ADD_MODE("-definerun", "", PUBTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-pending", "", PUBTOOL_MODE_PENDING, pendingArgs); Index: branches/simtest_nebulous_branches/ippTools/src/pxadd.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxadd.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/pxadd.c (revision 27840) @@ -0,0 +1,174 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE addRun SET state = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, add_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + + +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE addRun SET addRun.label = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, label, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *reduction, + char *label, + char *data_group, + char *dvodb, + char *note, + bool image_only) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("addtool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + + // queue the exp + // Note: cam_id is being cast here work around psS64 have a different type different + // on 32/64 + if (!p_psDBRunQueryF(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + reduction? reduction : "NULL", + label ? label : "NULL", + data_group ? data_group : "NULL", + dvodb ? dvodb : "NULL", + note ? note : "NULL", + image_only, + (long long) cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/pxadd.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxadd.h (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/pxadd.h (revision 27840) @@ -0,0 +1,42 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADD_H +#define PXADD_H 1 + +#include + +#include "pxtools.h" + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state); +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *reduction, + char *label, + char *data_group, + char *dvodb, + char *note, + bool image_only); + +#endif // PXADD_H Index: branches/simtest_nebulous_branches/ippTools/src/pxadmin.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxadmin.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxadmin.c (revision 27840) @@ -2,5 +2,5 @@ * pxadmin.c * - * Copyright (C) 2006-2008 Joshua Hoblitt + * Copyright (C) 2006-2009 Joshua Hoblitt * * This program is free software; you can redistribute it and/or modify it @@ -31,4 +31,5 @@ bool createMirrorMode(pxConfig *config); bool deleteMode(pxConfig *config); +static bool insert_dbversion(pxConfig * config, const char *versionString); static bool runMultipleStatments(pxConfig *config, const char *query); @@ -93,5 +94,5 @@ psString query = pxDataGet("pxadmin_create_tables.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -113,4 +114,9 @@ psFree(query); + if (!insert_dbversion(config, IPPDB_VERSION)) { + psError(PS_ERR_UNKNOWN, false, "failed to set database version"); + return false; + } + // COMMIT if (!psDBCommit(config->dbh)) { @@ -129,5 +135,5 @@ psString query = pxDataGet("pxadmin_create_mirror_tables.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -149,4 +155,9 @@ psFree(query); + if (!insert_dbversion(config, IPPDB_VERSION)) { + psError(PS_ERR_UNKNOWN, false, "failed to set database version"); + return false; + } + // COMMIT if (!psDBCommit(config->dbh)) { @@ -209,5 +220,5 @@ psString query = pxDataGet("pxadmin_drop_tables.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -262,2 +273,35 @@ return true; } + +#ifdef notdef +static bool update_dbversion(pxConfig * config, const char *versionString) +{ + psString query = pxDataGet("pxadmin_update_version.sql"); + if (!query) { + psError(PS_ERR_UNKNOWN, false, "failed to retrieve SQL statement"); + psFree(query); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, versionString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + return true; +} +#endif +static bool insert_dbversion(pxConfig * config, const char *versionString) +{ + psString query = "INSERT INTO dbversion VALUES('%s', CURRENT_TIMESTAMP())"; + if (!query) { + psError(PS_ERR_UNKNOWN, false, "failed to retrieve SQL statement"); + psFree(query); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, versionString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/pxcam.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxcam.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxcam.c (revision 27840) @@ -76,16 +76,16 @@ bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where) { - PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); - PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); @@ -141,86 +141,17 @@ } - -bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, - "invalid chipRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE camRun SET camRun.label = '%s' WHERE cam_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, cam_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for cam_id %" PRId64, cam_id); - return false; - } - - return true; -} - -bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - bool pxcamQueueByChipID(pxConfig *config, psS64 chip_id, char *workdir, char *label, + char *data_group, + char *dist_group, char *recipe, char *expgroup, char *dvodb, char *tess_id, - char *end_stage) + char *end_stage, + psS64 magicked, + char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -232,5 +163,5 @@ psMemSetPersistent(query, true); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -238,5 +169,5 @@ // queue the exp - // XXX chip_id is being cast here work around psS64 have a different type + // XXX chip_id and magicked are being cast here work around psS64 have a different type // different on 32/64 if (!p_psDBRunQueryF(config->dbh, query, @@ -245,4 +176,6 @@ "dirty", //workdir_state label ? label : "NULL", + data_group ? data_group : "NULL", + dist_group ? dist_group : "NULL", recipe ? recipe : "NULL", expgroup ? expgroup : "NULL", @@ -250,4 +183,6 @@ tess_id ? tess_id : "NULL", end_stage ? end_stage : "NULL", + (long long) magicked, + note ? note : "NULL", (long long)chip_id )) { Index: branches/simtest_nebulous_branches/ippTools/src/pxcam.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxcam.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxcam.h (revision 27840) @@ -26,8 +26,4 @@ bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state, psS64 magicked); -bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label); -bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - bool pxcamSetSearchArgs (psMetadata *md); bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where); @@ -37,9 +33,13 @@ char *workdir, char *label, + char *data_group, + char *dist_group, char *recipe, char *expgroup, char *dvodb, char *tess_id, - char *end_stage); + char *end_stage, + psS64 magicked, + char *note); #endif // PXCAM_H Index: branches/simtest_nebulous_branches/ippTools/src/pxchip.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxchip.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxchip.c (revision 27840) @@ -96,5 +96,5 @@ PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); @@ -125,9 +125,9 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); - return true; -} - -bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked) + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + return true; +} + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -140,5 +140,5 @@ } - char *query = "UPDATE chipRun SET state = '%s', magicked = %d WHERE chip_id = %" PRId64; + char *query = "UPDATE chipRun SET state = '%s', magicked = %" PRId64 " WHERE chip_id = %" PRId64; if (!p_psDBRunQueryF(config->dbh, query, state, magicked, chip_id)) { psError(PS_ERR_UNKNOWN, false, @@ -175,4 +175,9 @@ psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'"); + if (!strcmp(state, "cleaned") || !strcmp(state, "purged")) { + // if magicked is non-zero set it to -1 + psStringAppend(&query, ", chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"); + } + if (where && psListLength(where->list) > 0) { psString whereClause = psDBGenerateWhereSQL(where, NULL); @@ -192,27 +197,26 @@ } - -bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label) +bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) { PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE chipRun SET label = '%s' WHERE chip_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, chip_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for chip_id %" PRId64, chip_id); - return false; - } - - return true; -} - - -bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET label = '%s'"); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipProcessedImfile state: %s", state); + return false; + } + +/* if (!strcmp(state, "full")) { */ +/* // There are states that need to be met for a run to be set to full that we don't */ +/* // check here. */ +/* // for example all of the run's Imfiles must have chipProcessedImfile.data_state == "full" */ +/* // chipRun.magicked = (SUM(!chipProcessedImfile.magicked) = 0) */ +/* // so don't do allow setting the state to full */ +/* psError(PS_ERR_UNKNOWN, true, "cannot use -updaterun so set chipRun state to full"); */ +/* return false; */ +/* } */ + + psString query = psStringCopy("UPDATE chipProcessedImfile JOIN chipRun USING(chip_id) JOIN rawExp ON chipRun.exp_id = rawExp.exp_id SET data_state = '%s'"); if (where && psListLength(where->list) > 0) { @@ -220,7 +224,10 @@ psStringAppend(&query, " %s", whereClause); psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { psFree(query); psError(PS_ERR_UNKNOWN, false, "database error"); @@ -232,5 +239,4 @@ return true; } - psS64 pxchipQueueByExpTag(pxConfig *config, @@ -238,9 +244,12 @@ const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *reduction, const char *expgroup, const char *dvodb, const char *tess_id, - const char *end_stage) + const char *end_stage, + const char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -260,4 +269,6 @@ "dirty", // workdir_state label, + data_group, + dist_group, reduction, expgroup, @@ -265,5 +276,6 @@ tess_id, end_stage, - 0 // magicked + 0, // magicked + note ) ) { Index: branches/simtest_nebulous_branches/ippTools/src/pxchip.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxchip.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxchip.h (revision 27840) @@ -25,8 +25,8 @@ #include "pxtools.h" -bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked); -bool pxchipRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label); -bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + + +bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked); +bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); psS64 pxchipQueueByExpTag(pxConfig *config, @@ -34,9 +34,12 @@ const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *reduction, const char *expgroup, const char *dvodb, const char *tess_id, - const char *end_stage); + const char *end_stage, + const char *note); Index: branches/simtest_nebulous_branches/ippTools/src/pxdqstats.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxdqstats.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/pxdqstats.c (revision 27840) @@ -0,0 +1,103 @@ +/* pxdqstats.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdqstats.h" + +bool pxdqstatsSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); + psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + + return true; +} + +bool pxdqstatsGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/pxdqstats.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxdqstats.h (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/pxdqstats.h (revision 27840) @@ -0,0 +1,15 @@ +/* pxdqstats.h + */ + +#ifndef PXDQSTATS_H +#define PXDQSTATS_H 1 + +#include + +#include "pxtools.h" + +bool pxdqstatsSetSearchArgs (psMetadata *md); +bool pxdqstatsGetSearchArgs (pxConfig *config, psMetadata *where); + + +#endif // PXDQSTATS_H Index: branches/simtest_nebulous_branches/ippTools/src/pxerrors.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxerrors.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxerrors.c (revision 27840) @@ -29,13 +29,11 @@ switch (err) { case PXTOOLS_ERR_SYS: - return PS_EXIT_SYS_ERROR; + return PS_EXIT_SYS_ERROR; case PXTOOLS_ERR_CONFIG: - return PS_EXIT_CONFIG_ERROR; + return PS_EXIT_CONFIG_ERROR; case PXTOOLS_ERR_PROG: - return PS_EXIT_PROG_ERROR; - case PXTOOLS_ERR_DATA: - return PS_EXIT_DATA_ERROR; + return PS_EXIT_PROG_ERROR; default: - return PS_EXIT_UNKNOWN_ERROR; + return PS_EXIT_UNKNOWN_ERROR; } return PS_EXIT_UNKNOWN_ERROR; Index: branches/simtest_nebulous_branches/ippTools/src/pxfake.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxfake.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxfake.c (revision 27840) @@ -51,87 +51,16 @@ } - -bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, - "invalid fakeRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE fakeRun SET fakeRun.label = '%s' WHERE fake_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, fake_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for fake_id %" PRId64, fake_id); - return false; - } - - return true; -} - - -bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - psS64 pxfakeQueueByCamID(pxConfig *config, psS64 cam_id, char *workdir, char *label, + char *data_group, + char *dist_group, char *reduction, char *expgroup, char *dvodb, char *tess_id, - char *end_stage) + char *end_stage, + char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -145,5 +74,5 @@ psMemSetPersistent(query, true); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -157,4 +86,6 @@ workdir ? workdir : "NULL", label ? label : "NULL", + data_group ? data_group : "NULL", + dist_group ? dist_group : "NULL", reduction? reduction: "NULL", expgroup ? expgroup : "NULL", @@ -162,4 +93,5 @@ tess_id ? tess_id : "NULL", end_stage ? end_stage : "NULL", + note ? note : "NULL", (long long)cam_id )) { Index: branches/simtest_nebulous_branches/ippTools/src/pxfake.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxfake.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxfake.h (revision 27840) @@ -26,8 +26,4 @@ bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state); -bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label); -bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - psS64 pxfakeQueueByCamID(pxConfig *config, @@ -35,9 +31,12 @@ char *workdir, char *label, + char *data_group, + char *dist_group, char *reduction, char *expgroup, char *dvodb, char *tess_id, - char *end_stage); + char *end_stage, + char *note); Index: branches/simtest_nebulous_branches/ippTools/src/pxinject.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxinject.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxinject.c (revision 27840) @@ -137,7 +137,9 @@ PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false); PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); // insert with error flag state set to 0 (no errors) - if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL)) { + if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL, bytes, md5sum)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; Index: branches/simtest_nebulous_branches/ippTools/src/pxinjectConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxinjectConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxinjectConfig.c (revision 27840) @@ -61,4 +61,6 @@ psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-tmp_class_id", 0, "define the class ID (required)", NULL); psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-uri", 0, "define the URI (required)", NULL); + psMetadataAddS32(newImfileArgs, PS_LIST_TAIL, "-bytes", 0, "define the size of the file", 0); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-md5sum", 0, "define the size of the file", NULL); // -updatenewExp Index: branches/simtest_nebulous_branches/ippTools/src/pxmagic.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxmagic.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/pxmagic.c (revision 27840) @@ -0,0 +1,98 @@ +/* + * pxmagic.c + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmagic.h" + +bool pxmagicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState) +{ + psString queryFile = NULL; + psStringAppend(&queryFile, "magictool_restore_%s.sql", stage); + psString query_temp = pxDataGet(queryFile); + if (!query_temp) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement from %s", queryFile); + psFree(queryFile); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(queryFile); + + // psStringSubstittute fails on strings created by pxDataGet() + psString query = psStringCopy(query_temp); + // change the @NEW_STATE@ in the sql file to our state + if (!psStringSubstitute(&query, newState, "@NEW_STATE@")) { + psError(PS_ERR_UNKNOWN, false, "failed to substitute state string"); + return false; + } + + psStringAppend(&query, " AND %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + return true; +} + +bool pxmagicAddWhere(pxConfig *config, psString *out, psString table) +{ + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_CONFIG, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_CONFIG, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(out, " AND %s.magicked = 0", table); + } + if (destreaked) { + psStringAppend(out, " AND %s.magicked != 0", table); + } + // Note -magicked is handled by the caller. XXX: Why? + return true; +} + +void pxmagicAddArguments(psMetadata *md) +{ + psMetadataAddBool(md, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(md, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); + psMetadataAddS64(md, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); +} Index: branches/simtest_nebulous_branches/ippTools/src/pxmagic.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxmagic.h (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/pxmagic.h (revision 27840) @@ -0,0 +1,32 @@ +/* + * pxmagic.h + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMAGIC_H +#define PXMAGIC_H 1 + +#include + +#include "pxtools.h" + +extern bool pxmagicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState); + +extern bool pxmagicAddWhere(pxConfig *config, psString *string, psString table); +extern void pxmagicAddArguments(psMetadata *md); + +#endif // PXMAGIC_H Index: branches/simtest_nebulous_branches/ippTools/src/pxregister.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxregister.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxregister.c (revision 27840) @@ -58,2 +58,67 @@ return true; } + +bool pxrawExpSetState(pxConfig *config, psS64 exp_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "full", 5) == 0) + || (strncmp(state, "new", 4) == 0) + || (strncmp(state, "goto_compressed", 14) == 0) + || (strncmp(state, "compressed", 11) == 0) + || (strncmp(state, "goto_lossy", 11) == 0) + || (strncmp(state, "lossy", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid rawExp state: %s", state); + return false; + } + + char *query = "UPDATE rawExp SET state = '%s' WHERE exp_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} +bool pxrawImfileSetState(pxConfig *config, psS64 exp_id, const char *class_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "full", 5) == 0) + || (strncmp(state, "new", 4) == 0) + || (strncmp(state, "goto_compressed", 14) == 0) + || (strncmp(state, "compressed", 11) == 0) + || (strncmp(state, "goto_lossy", 11) == 0) + || (strncmp(state, "lossy", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid rawImfile data_state: %s", state); + return false; + } + + char *query = "UPDATE rawImfile SET data_state = '%s' WHERE class_id = %s AND exp_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, class_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/pxregister.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxregister.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxregister.h (revision 27840) @@ -26,4 +26,6 @@ bool pxnewExpSetState(pxConfig *config, psS64 exp_id, const char *state); +bool pxrawExpSetState(pxConfig *config, psS64 exp_id, const char *state); +bool pxrawImfileSetState(pxConfig *config, psS64 exp_id, const char *class_id, const char *state); #endif // PXREGISTER_H Index: branches/simtest_nebulous_branches/ippTools/src/pxspace.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxspace.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/pxspace.c (revision 27840) @@ -0,0 +1,57 @@ +/* + * pxmagic.c + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +// #include +// #include +// #include + +#include "pxtools.h" + +bool pxspaceAddWhere(pxConfig *config, psString *pQuery, psString table) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(table, NULL); + + PXOPT_LOOKUP_F64(radius, config->args, "-radius", false, false); + + if (radius > 0) { + PXOPT_LOOKUP_F64(ra, config->args, "-ra", false, false); + PXOPT_LOOKUP_F64(decl, config->args, "-decl", false, false); + + ra *= PS_RAD_DEG; + decl *= PS_RAD_DEG; + radius *= PS_RAD_DEG; + + psStringAppend(pQuery, " AND (ACOS((cos(%s.decl) * cos(%lf) * cos(%s.ra - %lf)) +(sin(%s.decl) * sin(%lf))) < %lf)", + table, decl, table, ra, table, decl, radius); + } + + return true; +} + +void pxspaceAddArguments(psMetadata *md) +{ + psMetadataAddF64(md, PS_LIST_TAIL, "-radius", 0, "search for exposures within radius RA DEC (degrees)", false); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra", 0, "RA value for radius search (degrees)", false); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl", 0, "DEC value for radius search (degrees)", false); +} Index: branches/simtest_nebulous_branches/ippTools/src/pxspace.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxspace.h (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/pxspace.h (revision 27840) @@ -0,0 +1,26 @@ +/* + * pxmagic.h + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXSPACE_H +#define PXSPACE_H 1 + +extern bool pxspaceAddWhere(pxConfig *config, psString *pQuery, psString table); +extern void pxspaceAddArguments(psMetadata *md); + +#endif // PXSPACE_H Index: branches/simtest_nebulous_branches/ippTools/src/pxtools.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxtools.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxtools.c (revision 27840) @@ -37,4 +37,6 @@ if (!strcmp(state, "goto_cleaned")) return true; if (!strcmp(state, "error_cleaned")) return true; + if (!strcmp(state, "goto_purged")) return true; + if (!strcmp(state, "error_purged")) return true; if (!strcmp(state, "goto_scrubbed")) return true; if (!strcmp(state, "error_scrubbed")) return true; @@ -42,12 +44,11 @@ if (!strcmp(state, "update")) return true; if (!strcmp(state, "purged")) return true; - if (!strcmp(state, "goto_purged")) return true; - if (!strcmp(state, "error_purged")) return true; - + if (!strcmp(state, "scrubbed")) return true; return false; } -// 'scrubbed' is a virtual state equivalent to cleaned, but allows files to be removed -// even if the config files is missing +// 'scrubbed' is no longer a virtual state equivalent to cleaned, but allows files to be removed +// even if the config files is missing. This change was prompted as files that are cleaned can +// be regenerated, but that is not certain after being scrubbed. @@ -60,5 +61,5 @@ if (!state) { psError(PS_ERR_PROGRAMMING, false, "%s not found in row %ld of table %s", - columnName, i, tableName); + columnName, i, tableName); return false; } @@ -69,5 +70,5 @@ // if state isn't cleaned or full we can't set it to cleaned psError(PS_ERR_PROGRAMMING, true, "%s with state %s may not be exported cleaned", - tableName, state); + tableName, state); return false; } @@ -89,20 +90,223 @@ psAssert (entry, "%s should at least have a place-holder", name); if (entry->data.str) { - psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true); - psMetadataItem *item = NULL; - while ((item = psListGetAndIncrement(iter))) { - // need to change the name and comment - psFree (item->name); - item->name = psStringCopy (field); - psFree (item->comment); - item->comment = psStringCopy (op); - if (!psMetadataAddItem(where, item, PS_LIST_TAIL, PS_META_DUPLICATE_OK)) { - psError(PS_ERR_UNKNOWN, false, "failed to add item %s", field); - psFree(where); - return false; - } - } - psFree (iter); - } - return true; -} + psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true); + psMetadataItem *item = NULL; + while ((item = psListGetAndIncrement(iter))) { + // need to change the name and comment + psFree (item->name); + item->name = psStringCopy (field); + psFree (item->comment); + item->comment = psStringCopy (op); + if (!psMetadataAddItem(where, item, PS_LIST_TAIL, PS_META_DUPLICATE_OK)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item %s", field); + psFree(where); + return false; + } + } + psFree (iter); + } + return true; +} + +// shared code for updating the various strings for a Run +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where, false); + PS_ASSERT_PTR_NON_NULL(pQuery, false); + PS_ASSERT_PTR_NON_NULL(*pQuery, false); + PS_ASSERT_PTR_NON_NULL(runTable, false); + PS_ASSERT_PTR_NON_NULL(idColumn, false); + PS_ASSERT_PTR_NON_NULL(fileTable, false); + + // make sure that -state is not the only selection parameter + PXOPT_LOOKUP_STR(where_state, config->args, "-state", false, false); + if (where_state && (psListLength(where->list) < 2)) { + psError(PXTOOLS_ERR_CONFIG, true, "selection by -state alone is not allowed"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + psString dist_group = NULL; + if (has_dist_group) { + PXOPT_LOOKUP_STR(tmp_dist_group, config->args, "-set_dist_group", false, false); + dist_group = tmp_dist_group; + } + + if ((!state) && (!label) && (!data_group) && (has_dist_group && !dist_group) && !(note)) { + psError(PXTOOLS_ERR_CONFIG, false, "parameters are required"); + return false; + } + + if (state && ! pxIsValidState(state)) { + psError(PXTOOLS_ERR_CONFIG, false, "pxIsValidState failed"); + return false; + } + + // first paramter is added with "SET param = 'value'" + // subseqent ones with ", param = 'value'" + char *separator = " SET "; + char *comma = ","; + +# define addColumn(_tab, _val) \ + do { \ + if (_val) { \ + psStringAppend(pQuery, "%s %s.%s = '%s'", separator, _tab, #_val, _val); \ + separator = comma; \ + } \ + } while (0) + + addColumn(runTable, state); + addColumn(runTable, data_group); + if (has_dist_group) { + addColumn(runTable, dist_group); + } + addColumn(runTable, note); + addColumn(runTable, label); + + psString joinHook = psStringCopy(""); + psString fileWhere = NULL; + if (state && !strcmp(state, "update")) { + psStringAppend(&joinHook, "\n JOIN %s USING(%s)", fileTable, idColumn); + psStringAppend(pQuery, ", %s.data_state = 'update'", fileTable); + psStringAppend(&fileWhere, "AND %s.data_state = 'cleaned'", fileTable); + } + + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(pQuery, " %s", whereClause); + psFree(whereClause); + if (fileWhere) { + psStringAppend(pQuery, "%s", fileWhere); + } + + bool mdok; // Status of MD lookup + if (psMetadataLookupBool(&mdok, config->args, "-pretend")) { + psLogMsg("pxtools", PS_LOG_INFO, "Query to run: %s\n", *pQuery); + return true; + } + + if (!p_psDBRunQueryF(config->dbh, *pQuery, joinHook)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool pxLookupVersion(pxConfig *config, psArray **pArray) +{ + const char *query = "SELECT * FROM dbversion"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, true, "no rows in dbversion"); + return false; + } + if (psArrayLength(output) > 1) { + psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found in dbversion: %ld", + psArrayLength(output)); + return false; + } + *pArray = output; + + return true; +} + +psString pxGetDBVersion(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psArray *array = NULL; + if (!pxLookupVersion(config, &array)) { + psError(PS_ERR_UNKNOWN, false, "pxLookupVersion failed"); + return NULL; + } + psMetadata *md = array->data[0]; + if (!md) { + psError(PS_ERR_UNKNOWN, true, "output of pxLookupVersion is null"); + return NULL; + } + + psString version = psMetadataLookupStr(NULL, md, "schema_version"); + + return version; +} + +bool pxExportVersion(pxConfig *config, FILE *file) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(file, NULL); + + psArray *array = NULL; + if (!pxLookupVersion(config, &array) || !array) { + psError(PS_ERR_UNKNOWN, false, "pxLookupVersion failed"); + return false; + } + if (!ippdbPrintMetadatas(file, array, "dbversion", true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(array); + return false; + } + return true; +} + +bool pxCheckImportVersion(pxConfig *config, psMetadata *input) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(input, NULL); + + // This code was adapted from the way camtool parses the structures. + // Is this really the way to do it? + psMetadataItem *multi_item = psMetadataLookup(input, "dbversion"); + if (!multi_item || (multi_item->type != PS_DATA_METADATA_MULTI)) { + psError(PS_ERR_UNKNOWN, true, "dbversion multi not found in input"); + return false; + } + + psMetadataItem *dbversion = psListGet(multi_item->data.list, 0); + if (!dbversion) { + psError(PS_ERR_UNKNOWN, true, "dbversion not found in input"); + return false; + } + + if (!strcmp(dbversion->name, "dbversion")) { + // horray + psMetadata *md = dbversion->data.md; + psString schema_version = pxGetDBVersion(config); + if (!schema_version) { + psError(PS_ERR_UNKNOWN, false, "pxGetDBVersion failed"); + return false; + } + + psString import_version = psMetadataLookupStr(NULL, md, "schema_version"); + if (import_version && strcmp(import_version, schema_version)) { + psError(PS_ERR_UNKNOWN, true, "input file schema_version: %s does not match data base: %s", + import_version, schema_version); + return false; + } else if (!import_version) { + psError(PS_ERR_UNKNOWN, true, "input file schema_version is NULL"); + return false; + } else { + // YIPPEE this file is the same version + } + } else { + psError(PS_ERR_UNKNOWN, true, "Unexpected config dump format"); + return false; + } + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/pxtools.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxtools.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxtools.h (revision 27840) @@ -37,4 +37,5 @@ #include "pxtoolsErrorCodes.h" +#include "pxadd.h" #include "pxcam.h" #include "pxchip.h" @@ -45,4 +46,6 @@ #include "pxtag.h" #include "pxtree.h" +#include "pxmagic.h" +#include "pxspace.h" # define MAX_ROWS 10e9 @@ -54,4 +57,6 @@ bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code); +bool pxExportVersion(pxConfig *config, FILE *f); +bool pxCheckImportVersion(pxConfig *config, psMetadata *md); psExit pxerrorGetExitStatus(void); @@ -59,4 +64,6 @@ void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet); bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets); + +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group); #define PXOPT_ADD_MODE(option, comment, modeval, argset) \ Index: branches/simtest_nebulous_branches/ippTools/src/pxtoolsErrorCodes.dat =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxtoolsErrorCodes.dat (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxtoolsErrorCodes.dat (revision 27840) @@ -8,3 +8,2 @@ CONFIG Problem in configure files PROG Programming error -DATA invalid data Index: branches/simtest_nebulous_branches/ippTools/src/pxwarp.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxwarp.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxwarp.c (revision 27840) @@ -29,108 +29,15 @@ #include "pxwarp.h" -bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state); - return false; - } - - char *query = "UPDATE warpRun SET state = '%s' WHERE warp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, warp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for warp_id %" PRId64, warp_id); - return false; - } - - return true; -} - - -bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - if (whereClause && strlen(whereClause) > 0) { - psStringAppend(&query, " %s", whereClause); - } - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE warpRun SET warpRun.label = '%s' WHERE warp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, warp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change label for warp_id %" PRId64, warp_id); - return false; - } - - return true; -} - - -bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - bool pxwarpQueueByFakeID(pxConfig *config, psS64 fake_id, const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *dvodb, const char *tess_id, const char *reduction, - const char *end_stage) + const char *end_stage, + const char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -145,10 +52,13 @@ "dirty", // workdir_state label, + data_group, + dist_group, dvodb, tess_id, - reduction, + reduction, end_stage, NULL, // registered - 0 // magicked zero when created will get updated when warpRun goes to 'full' + 0, // magicked set to zero when created may get updated when warpRun goes to 'full' + NULL // note )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/simtest_nebulous_branches/ippTools/src/pxwarp.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pxwarp.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pxwarp.h (revision 27840) @@ -25,17 +25,15 @@ #include "pxtools.h" -bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state); -bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label); -bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - bool pxwarpQueueByFakeID(pxConfig *config, psS64 fake_id, const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *dvodb, const char *tess_id, const char *reduction, - const char *end_stage); + const char *end_stage, + const char *note); #endif // PXWARP_H Index: branches/simtest_nebulous_branches/ippTools/src/pztool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pztool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pztool.c (revision 27840) @@ -117,5 +117,6 @@ telescope, uri, - NULL // epoch + NULL, // epoch + 0 // use_compress )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -304,5 +305,5 @@ psArray *cameras = pzGetPendingCameras(config); if (!cameras) { - psError(PXTOOLS_ERR_DATA, false, "failed to find any cameras"); + psError(PXTOOLS_ERR_CONFIG, false, "failed to find any cameras"); return false; } @@ -314,5 +315,5 @@ psString query = pxDataGet("pztool_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(cameraImfiles); return false; @@ -401,4 +402,6 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_BOOL(row_lock, config->args, "-row_lock", false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); // default values @@ -457,5 +460,7 @@ fault, NULL, // epoch - hostname + hostname, + bytes, + md5sum )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -488,5 +493,5 @@ psString query = pxDataGet("pztool_find_completed_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -595,5 +600,7 @@ " pzDownloadImfile.class_id," // tmp_class_id " pzDownloadImfile.uri," // uri - " NULL" // epoch + " NULL," // epoch + " pzDownloadImfile.bytes," // bytes + " pzDownloadImfile.md5sum" // md5sum " FROM pzDownloadImfile" " WHERE" @@ -777,5 +784,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); @@ -801,5 +808,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "inst", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); @@ -808,5 +815,5 @@ psString query = pxDataGet("pztool_revertcopied.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -842,5 +849,5 @@ psString query = pxDataGet("pztool_revert_downloadimfile_faults.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -857,5 +864,5 @@ psString query = pxDataGet("pztool_revert_fileset_faults.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } Index: branches/simtest_nebulous_branches/ippTools/src/pztoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/pztoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/pztoolConfig.c (revision 27840) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (! config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!\n"); + psError(psErrorCodeLast(), false, "Can't find site configuration!\n"); psFree(config); return NULL; @@ -95,4 +95,6 @@ psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-hostname", 0, "define the host that copied the image", NULL); + psMetadataAddS32(copydoneArgs, PS_LIST_TAIL, "-bytes", 0, "define the size in bytes for the copied image", 0); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-md5sum", 0, "define the md5 sum for the copied image", NULL); psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0, "lock pzDownImfile rows while advancing an exposure", false); Index: branches/simtest_nebulous_branches/ippTools/src/receivetool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/receivetool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/receivetool.c (revision 27840) @@ -105,4 +105,5 @@ // optional PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(last, config->args, "-last", false, false); PXOPT_LOOKUP_STR(status_product, config->args, "-status_product", false, false); @@ -110,5 +111,5 @@ PXOPT_LOOKUP_STR(ds_dbhost, config->args, "-ds_dbhost", false, false); - if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, comment, last, status_product, ds_dbname, ds_dbhost)) { + if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, state, comment, last, status_product, ds_dbname, ds_dbhost)) { psError(PS_ERR_UNKNOWN, false, "Database error"); return false; @@ -135,5 +136,5 @@ psString query = pxDataGet("receivetool_list.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -142,5 +143,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -194,5 +195,5 @@ psString query = pxDataGet("receivetool_addfileset.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); return false; } @@ -301,5 +302,5 @@ psString query = pxDataGet("receivetool_pendingfileset.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -380,5 +381,5 @@ psString file_type = psMetadataLookupStr(NULL, md, "file_type"); psString component = psMetadataLookupStr(NULL, md, "component"); - + if (!file) { psError(PS_ERR_UNKNOWN, false, "failed to find value for file"); @@ -430,5 +431,5 @@ psString query = pxDataGet("receivetool_pendingfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -513,5 +514,5 @@ psString query = pxDataGet("receivetool_revert.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -546,5 +547,5 @@ psString query = pxDataGet("receivetool_toadvance.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "Failed to retreive SQL statement"); psFree(where); return false; @@ -612,5 +613,5 @@ psString query = NULL; // Query to execute psStringAppend(&query, "UPDATE receiveFileset SET "); - + psString sep = ""; if (fault) { @@ -630,5 +631,5 @@ sep = ","; } - + psStringAppend(&query, " WHERE fileset_id = %" PRId64, fileset_id); Index: branches/simtest_nebulous_branches/ippTools/src/receivetoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/receivetoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/receivetoolConfig.c (revision 27840) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!"); + psError(psErrorCodeLast(), false, "Can't find site configuration!"); psFree(config); return NULL; @@ -49,4 +49,5 @@ psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-state", 0, "define state", "enabled"); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-last", 0, "define last fileset", NULL); psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-status_product", 0, "define status_product", NULL); Index: branches/simtest_nebulous_branches/ippTools/src/regtool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/regtool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/regtool.c (revision 27840) @@ -37,4 +37,5 @@ static bool revertprocessedimfileMode(pxConfig *config); static bool updateprocessedimfileMode(pxConfig *config); +static bool pendingcompressimfileMode(pxConfig *config); // exp static bool pendingexpMode(pxConfig *config); @@ -44,4 +45,8 @@ static bool updateprocessedexpMode(pxConfig *config); static bool cleardupexpMode(pxConfig *config); +static bool finishcompressexpMode(pxConfig *config); + +static bool updatebyqueryMode(pxConfig *config); + static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -77,4 +82,7 @@ MODECASE(REGTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(REGTOOL_MODE_UPDATEBYQUERY, updatebyqueryMode); + MODECASE(REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileMode); + MODECASE(REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpMode); MODECASE(REGTOOL_MODE_CLEARDUPEXP, cleardupexpMode); MODECASE(REGTOOL_MODE_EXPORTRUN, exportrunMode); @@ -117,5 +125,5 @@ psString query = pxDataGet("regtool_pendingimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -189,4 +197,5 @@ PXOPT_LOOKUP_STR(obs_mode, config->args, "-obs_mode", false, false); PXOPT_LOOKUP_STR(obs_group, config->args, "-obs_group", false, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false); PXOPT_LOOKUP_F32(airmass, config->args, "-airmass", false, false); @@ -224,4 +233,5 @@ PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false); PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false); + PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); @@ -238,4 +248,6 @@ PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -252,4 +264,5 @@ class_id, uri, + data_state, exp_type, filelevel, @@ -307,5 +320,8 @@ quality, NULL, - 0 + 0, + bytes, + md5sum, + 0 // burntool_state )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -327,12 +343,33 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "=="); + PXOPT_COPY_STR(config->args, where, "-data_state", "data_state", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(allfiles, config->args, "-allfiles", false); + if (allfiles) { + faulted = false; + } PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false); + + // build where string for some parameters that don't fit PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "rawImfile")) { + psError(psErrorCodeLast(), false, "pxmagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawImfile")) { + psError(psErrorCodeLast(), false, "pxspaceAddWhere failed"); + return false; + } psString query = pxDataGet("regtool_processedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -343,13 +380,25 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; } psFree(where); + if (where2) { + psStringAppend(&query, " %s", where2); + psFree(where2); + } if (faulted) { // list only faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault != 0"); - } else { + } else if (!allfiles) { // don't list faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); + } + + // add the ORDER BY statement if desired + if (ordered_by_date) { + psStringAppend(&query, " ORDER BY dateobs"); } @@ -417,5 +466,5 @@ psString query = pxDataGet("regtool_revertprocessedimfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -428,5 +477,5 @@ } else { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -456,12 +505,15 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); - - if ((fault == INT16_MAX) && !isfinite(user_1)) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -user_1 must be selected"); - return false; - } - if ((fault != INT16_MAX) && isfinite(user_1)) { - psError(PS_ERR_UNKNOWN, false, "only one of -fault or -user_1 must be selected"); + PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); + PXOPT_LOOKUP_S32(set_bytes, config->args, "-set_bytes", false, false); + PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + + if ((fault == INT16_MAX) && !isfinite(burntool_state) && !(set_state)) { + psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); + return false; + } + if ((fault != INT16_MAX) && isfinite(burntool_state)) { + psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); return false; } @@ -472,29 +524,141 @@ if (fault != INT16_MAX) { - // this is fairly dangerous : can set all if the where is not set... - if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - psFree (where); - return false; - } - } - - if (isfinite(user_1)) { - psString query = pxDataGet("regtool_updateprocessedimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, user_1, exp_id, class_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); + // this is fairly dangerous : can set all if the where is not set... + if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return(true); } psFree (where); + psString setvalues = NULL; + if (burntool_state != INT16_MAX) { + psStringAppend(&setvalues,"rawImfile.burntool_state = %d",burntool_state); + } + if (set_bytes != INT32_MAX) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.bytes = %d",set_bytes); + } + if (set_md5sum) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.md5sum = '%s'",set_md5sum); + } + if (set_state) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state); + } + + psString query = pxDataGet("regtool_updateprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // printf(query,setvalues,exp_id,class_id); + if (!p_psDBRunQueryF(config->dbh, query, setvalues, exp_id,class_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(setvalues); + return false; + } + psFree(setvalues); + psFree(query); + return true; +} + +static bool pendingcompressimfileMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (whereClause) { + psStringPrepend(&whereClause, "\n AND "); + } + PXOPT_LOOKUP_BOOL(compress, config->args, "-compress", false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + if ((compress && clean) || (!compress && !clean)) { + psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')\n OR (data_state = 'goto_lossy' AND state = 'goto_lossy')) "); + } + else if (compress) { + psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')) "); + } + else if (clean) { + psStringAppend(&whereClause, "\n AND ((data_state = 'goto_lossy' AND state = 'goto_lossy')) "); + } + else { + psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')\n OR (data_state = 'goto_lossy' AND state = 'goto_lossy')) "); + } + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_pendingcompressimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + // printf(query,whereClause,limitString); + if (!p_psDBRunQueryF(config->dbh, query, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingCompressImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; } @@ -513,5 +677,5 @@ psString query = pxDataGet("regtool_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -590,4 +754,5 @@ PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); @@ -651,5 +816,5 @@ psString query = pxDataGet("regtool_pendingexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -751,4 +916,5 @@ filelevel, workdir, + state, reduction, dvodb, @@ -849,9 +1015,12 @@ workdir, label, + NULL, // data_group + NULL, // dist_group reduction, - NULL, // expgroup + NULL, // expgroup dvodb, tess_id, - end_stage + end_stage, + NULL // note )) { // rollback @@ -881,5 +1050,5 @@ psString query = pxDataGet("regtool_processedexp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -891,11 +1060,12 @@ PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">="); PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<"); @@ -926,4 +1096,14 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxmagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -931,28 +1111,33 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); } psFree(where); + if (where2) { + psStringAppend(&query, " %s ", where2); + psFree(where2); + } + if (faulted) { - // list only faulted rows - psStringAppend(&query, " %s", "AND rawExp.fault != 0"); + // list only faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault != 0"); } else { - // don't list faulted rows - psStringAppend(&query, " %s", "AND rawExp.fault = 0"); + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault = 0"); } // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; } psFree(query); @@ -960,29 +1145,29 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; } if (!psArrayLength(output)) { - psTrace("regtool", PS_LOG_INFO, "no rows found"); + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } + return false; + } } @@ -994,42 +1179,42 @@ static bool revertprocessedexpMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id_begin", "exp_id", ">="); - PXOPT_COPY_S64(config->args, where, "-exp_id_end", "exp_id", "<="); - - psString query = pxDataGet("regtool_revertprocessedexp.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - psFree(where); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } else { - psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id_begin", "exp_id", ">="); + PXOPT_COPY_S64(config->args, where, "-exp_id_end", "exp_id", "<="); + + psString query = pxDataGet("regtool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - - int numUpdated = psDBAffectedRows(config->dbh); - - psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated); - - return true; + return false; + } + psFree(query); + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated); + + return true; } @@ -1037,20 +1222,207 @@ static bool updateprocessedexpMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - - if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - psFree(where); - return false; - } + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree(where); - - return true; -} + return false; + } + psFree(where); + + return true; +} + + +static bool finishcompressexpMode(pxConfig *config) { + + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_finishcompressexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " %s ", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s ", limitString); + psFree(limitString); + } + // printf(query); + // return(false); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + rawExpRow *row = rawExpObjectFromMetadata(md); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (strcmp(row->state,"goto_compressed") == 0) { + if (!pxrawExpSetState(config,row->exp_id, "compressed")) { + psError(PS_ERR_UNKNOWN, false, "failed to set rawExp.state for exp_id: %" PRId64, row->exp_id); + psFree(row); + psFree(output); + return(false); + } + } + else if (strcmp(row->state,"goto_lossy") == 0) { + if (!pxrawExpSetState(config,row->exp_id, "lossy")) { + psError(PS_ERR_UNKNOWN, false, "failed to set rawExp.state for exp_id: %" PRId64, row->exp_id); + psFree(row); + psFree(output); + return(false); + } + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(row); + } + + psFree(output); + return(true); +} + +static bool updatebyqueryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", true, false); + + // XX test this out; need to make this consistent with the list in regtoolConfig.c + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "rawExp.state", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause,"\n AND "); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + // Update the imfiles first, because if you select by state, you'll clobber things + psString query = pxDataGet("regtool_updatebyqueryimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + + // printf(query,set_state,whereClause,limitString); + if (!p_psDBRunQueryF(config->dbh, query, set_state, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + + psFree(query); + // Now up date the exposure. + query = pxDataGet("regtool_updatebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, set_state,whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + + return true; +} + @@ -1072,5 +1444,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1098,5 +1470,5 @@ psError(PS_ERR_UNKNOWN, false, "database error"); } - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1179,4 +1551,9 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion to output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); @@ -1190,5 +1567,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1248,4 +1625,10 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + fprintf (stdout, "---- input ----\n"); Index: branches/simtest_nebulous_branches/ippTools/src/regtool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/regtool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/regtool.h (revision 27840) @@ -36,4 +36,7 @@ REGTOOL_MODE_UPDATEPROCESSEDEXP, REGTOOL_MODE_CLEARDUPEXP, + REGTOOL_MODE_UPDATEBYQUERY, + REGTOOL_MODE_PENDINGCOMPRESSIMFILE, + REGTOOL_MODE_FINISHCOMPRESSEXP, REGTOOL_MODE_EXPORTRUN, REGTOOL_MODE_IMPORTRUN Index: branches/simtest_nebulous_branches/ippTools/src/regtoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/regtoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/regtoolConfig.c (revision 27840) @@ -43,5 +43,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PXTOOLS_ERR_CONFIG, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -72,6 +72,6 @@ ADD_OPT(Str, addprocessedimfileArgs, "-comment", "define comment ", NULL); ADD_OPT(Str, addprocessedimfileArgs, "-obs_mode", "define observing mode (data usage goal)", NULL); - ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); - + ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-data_state", "define the data_state", "full"); // Note: RA & DEC are supplied here in radians, but the query options use degrees. This // can be justified by the fact that this option (addprocessedimfile) is a software @@ -125,4 +125,7 @@ ADD_OPT(Time, addprocessedimfileArgs, "-dateobs", "define observation time", NULL); ADD_OPT(Str, addprocessedimfileArgs, "-hostname", "define host name", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-md5sum", "define md5sum", NULL); + ADD_OPT(S32, addprocessedimfileArgs, "-bytes", "define bytes", 0); + ADD_OPT(S16, addprocessedimfileArgs, "-burntool_state", "set burntool state", 0); ADD_OPT(S16, addprocessedimfileArgs, "-fault", "set fault code", 0); ADD_OPT(S16, addprocessedimfileArgs, "-quality", "set quality flag", 0); @@ -133,10 +136,18 @@ ADD_OPT(Str, processedimfileArgs, "-exp_name", "search by exposure name", NULL); ADD_OPT(Str, processedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(Str, processedimfileArgs, "-filter", "search by filter", NULL); + ADD_OPT(Str, processedimfileArgs, "-obs_mode", "search by obs_mod", NULL); + ADD_OPT(Str, processedimfileArgs, "-exp_type", "search by exposure type", NULL); + ADD_OPT(Str, processedimfileArgs, "-data_state", "search by data_state", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL); ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); + ADD_OPT(Bool, processedimfileArgs, "-allfiles", "return imfiles regardless of fault status", false); + ADD_OPT(Bool, processedimfileArgs, "-all", "list without search arguments", false); ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); ADD_OPT(Bool, processedimfileArgs, "-ordered_by_date", "force output to be in DATE order", false); + pxspaceAddArguments(processedimfileArgs); + pxmagicAddArguments(processedimfileArgs); // -revertprocessedimfile @@ -153,7 +164,11 @@ ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); - ADD_OPT(F64, updateprocessedimfileArgs, "-user_1", "set user stat (1)", NAN); + ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", INT16_MAX); ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); - + ADD_OPT(S32, updateprocessedimfileArgs, "-set_bytes", "set bytes", INT32_MAX); + ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum", "set md5sum", NULL); + ADD_OPT(Str, updateprocessedimfileArgs, "-set_state", "set data state", NULL); + + // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); @@ -176,4 +191,5 @@ ADD_OPT(Str, addprocessedexpArgs, "-exp_type", "define exposure type", NULL); ADD_OPT(Str, addprocessedexpArgs, "-workdir", "define the \"default\" workdir for this exposure", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-state", "define the state for this exposure", NULL); ADD_OPT(Str, addprocessedexpArgs, "-dvodb", "define the dvodb for the next processing step", NULL); ADD_OPT(Str, addprocessedexpArgs, "-tess_id", "define the tess_id for the next processing step", NULL); @@ -244,4 +260,5 @@ psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-state", 0, "search by exposure state", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); @@ -276,4 +293,6 @@ psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + pxspaceAddArguments(processedexpArgs); + pxmagicAddArguments(processedexpArgs); psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -292,5 +311,5 @@ psMetadataAddS64(updatedprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", INT16_MAX); - + psMetadataAddStr(updatedprocessedexpArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -303,8 +322,73 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - // -cleardupexp psMetadata *cleardupexpArgs = psMetadataAlloc(); + // -updatebyquery + psMetadata *updatebyqueryArgs = psMetadataAlloc(); + + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "set the state", NULL); + + psMetadataAddS64(updatebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(updatebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(updatebyqueryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-state", 0, "search by rawExp.state", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by obs_mode", NULL); + + psMetadataAddU64(updatebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(updatebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + + // -pendingcompressimfile + psMetadata *pendingcompressimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcompressimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(pendingcompressimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcompressimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-compress", 0, "search only for files to compress", 0); + psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-clean", 0, "search only for files to clean originals", 0); + + // -finishcompressexp + psMetadata *finishcompressexpArgs = psMetadataAlloc(); + psMetadataAddS64(finishcompressexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddBool(finishcompressexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(finishcompressexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -321,4 +405,7 @@ PXOPT_ADD_MODE("-updateprocessedexp", "", REGTOOL_MODE_UPDATEPROCESSEDEXP, updatedprocessedexpArgs); PXOPT_ADD_MODE("-cleardupexp", "", REGTOOL_MODE_CLEARDUPEXP, cleardupexpArgs); + PXOPT_ADD_MODE("-updatebyquery", "", REGTOOL_MODE_UPDATEBYQUERY, updatebyqueryArgs); + PXOPT_ADD_MODE("-pendingcompressimfile", "", REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileArgs); + PXOPT_ADD_MODE("-finishcompressexp", "", REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", REGTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", REGTOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/simtest_nebulous_branches/ippTools/src/stacktool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/stacktool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/stacktool.c (revision 27840) @@ -109,8 +109,14 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); psMetadata *where = psMetadataAlloc(); @@ -118,4 +124,5 @@ // select based on properties of the raw exposures + PXOPT_COPY_STR(config->args, where, "-select_comment", "rawExp.comment", "LIKE"); PXOPT_COPY_STR(config->args, where, "-select_inst", "rawExp.camera", "=="); PXOPT_COPY_STR(config->args, where, "-select_telescope", "rawExp.telescope", "=="); @@ -139,11 +146,26 @@ PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_min", "camProcessedExp.fwhm_minor", ">="); PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_max", "camProcessedExp.fwhm_minor", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_max", "camProcessedExp.iq_m2", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_min", "camProcessedExp.iq_m2", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m3_max", "camProcessedExp.iq_m3", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_min", "camProcessedExp.iq_m4", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_max", "camProcessedExp.iq_m4", "<="); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "warpSkyfile.skycell_id", "=="); - pxAddLabelSearchArgs (config, where, "-select_label", "warpRun.label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-select_data_group", "warpRun.data_group", "=="); + pxAddLabelSearchArgs (config, where, "-select_label", "warpRun.label", "LIKE"); // define using warp label // these are used to build the HAVING restriction PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">="); + PXOPT_LOOKUP_S32(min_num, config->args, "-min_num", false, false); + if (min_num < 2) { + psError(PXTOOLS_ERR_CONFIG, true, "Require at least two inputs for a stack, but min_num = %d", + min_num); + psFree(where); + psFree(having); + return false; + } // other options applied outside of the WHERE @@ -154,33 +176,31 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - psString select = pxDataGet("stacktool_definebyquery_part1.sql"); + psFree(having); + return false; + } + + psString select = pxDataGet("stacktool_definebyquery_select.sql"); if (!select) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + psFree(having); + return false; + } + + psString where1 = psStringCopy(""); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&select, " AND %s", whereClause); + psStringAppend(&where1, "\nAND %s", whereClause); psFree(whereClause); } - psString groupby = pxDataGet("stacktool_definebyquery_part2.sql"); - if (!groupby) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - psFree(where); - return false; - } - psStringAppend(&select, " %s", groupby); - psFree(groupby); + psString where2 = psStringCopy(""); + if (label) { + psStringAppend(&where2, "\nWHERE stackRun.label = '%s'", label); + } // Restriction on aggregated quantities using HAVING @@ -214,11 +234,14 @@ psFree(having); - if (!p_psDBRunQuery(config->dbh, select)) { + if (!p_psDBRunQueryF(config->dbh, select, where1, where2)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(select); - psFree(where); + psFree(where1); + psFree(where2); return false; } psFree(select); + psFree(where1); + psFree(where2); psArray *output = p_psDBFetchResult(config->dbh); @@ -262,5 +285,5 @@ } if (!insert) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -277,5 +300,5 @@ psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql"); if (!part2) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(insert); return false; @@ -339,4 +362,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, dvodb, @@ -344,5 +369,6 @@ skycell_id, tess_id, - filter); + filter, + note); if (!stackRunInsertObject(config->dbh, run)) { @@ -462,5 +488,5 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); @@ -469,10 +495,13 @@ // default PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); // options - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); // we have to support multipe exp_ids @@ -489,4 +518,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, dvodb, @@ -494,5 +525,6 @@ skycell_id, tess_id, - filter); + filter, + note); if (!run) { @@ -563,13 +595,44 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); +#ifdef notdef + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); +#endif + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE stackRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true); + + psFree(query); + psFree(where); + + return result; + +#ifdef notdef + // Hack-y work around to make stacktool more like the other tools, without breaking other stuff (hopefully). + + if ((state)&&(stack_id)) { // set detRun.state to state return setstackRunState(config, stack_id, state); } - return true; + if ((state)&&(label)) { + return setstackRunStateByLabel(config, label, state); + } + + psError(PS_ERR_UNKNOWN, false, "Required options not found."); + return false; +#endif } @@ -613,5 +676,5 @@ psString query = pxDataGet("stacktool_inputskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -619,5 +682,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile"); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); } @@ -687,12 +750,13 @@ psString query = pxDataGet("stacktool_tosum.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psStringCopy(""); // WHERE conditions to add if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + psString new = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\nAND %s", new); + psFree(new); } psFree(where); @@ -705,5 +769,5 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -853,10 +917,18 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); + +// The following three selectors are incompatible with the sql so omit them +// PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); +// PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); +// PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -865,5 +937,5 @@ psString query = pxDataGet("stacktool_sumskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -873,5 +945,9 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); @@ -930,10 +1006,10 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -942,5 +1018,5 @@ psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); if (!delete) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -989,4 +1065,26 @@ } +#ifdef notdef +static bool setstackRunStateByLabel(pxConfig *config, const char *label, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE label = '%s'"; + if (!p_psDBRunQueryF(config->dbh, query, state, label)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for label %s", label); + return false; + } + + return true; +} +#endif + static bool pendingcleanuprunMode(pxConfig *config) { @@ -1001,5 +1099,5 @@ psString query = pxDataGet("stacktool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1065,5 +1163,5 @@ psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1126,5 +1224,5 @@ psString query = pxDataGet("stacktool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1214,4 +1312,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); @@ -1226,5 +1328,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1300,7 +1402,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "stackRun"); psAssert (item, "entry not in input?"); Index: branches/simtest_nebulous_branches/ippTools/src/stacktoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/stacktoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/stacktoolConfig.c (revision 27840) @@ -38,5 +38,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -47,13 +47,18 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_exp_type", 0, "search for exp_type", "object"); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by warpRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_data_group", 0 , "search by warpRun data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_comment", 0, "search for comment (LIKE)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_inst", 0, "search for camera", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_telescope", 0, "search for telescope", NULL); @@ -77,6 +82,11 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_min", 0, "define min fwhm (minor axis)", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_max", 0, "define max fwhm (minor axis)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_min", 0, "define min iq_m2", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_max", 0, "define max iq_m2", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_min", 0, "define min iq_m3", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_max", 0, "define max iq_m3", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_min", 0, "define min iq_m4", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_max", 0, "define max iq_m4", NAN); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_new", 0, "minimum number of new inputs", 0); @@ -87,9 +97,9 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); @@ -100,6 +110,12 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -addinputskyfile @@ -158,16 +174,24 @@ psMetadata *sumskyfileArgs= psMetadataAlloc(); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); - psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); +#ifdef notdef + // These don't work so omit (for now) We probably should create a different mode for this type of search. + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); +#endif + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by stackRun.label", NULL); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-data_group", 0, "search by stackRun.data_group (LIKE comparison)", NULL); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", NULL); + psMetadataAddS16(sumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-all", 0, "enable search without arguments", false); // -revertsumskyfile psMetadata *revertsumskyfileArgs= psMetadataAlloc(); psMetadataAddS64(revertsumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); - psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); psMetadataAddS16(revertsumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddBool(revertsumskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); Index: branches/simtest_nebulous_branches/ippTools/src/staticskytool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/staticskytool.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/staticskytool.c (revision 27840) @@ -0,0 +1,1075 @@ +/* + * stacktool.c + * + * Copyright (C) 2007-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "stacktool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool definerunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool addinputskyfileMode(pxConfig *config); +static bool inputskyfileMode(pxConfig *config); +static bool tosumMode(pxConfig *config); +static bool addsumskyfileMode(pxConfig *config); +static bool sumskyfileMode(pxConfig *config); +static bool revertsumskyfileMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupskyfileMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool updatesumskyfileMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); + +static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = stacktoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(STACKTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(STACKTOOL_MODE_INPUTS, inputsMode); + MODECASE(STACKTOOL_MODE_TODO, todoMode); + MODECASE(STACKTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(STACKTOOL_MODE_RESULT, resultMode); + MODECASE(STACKTOOL_MODE_REVERT, revertMode); + MODECASE(STACKTOOL_MODE_UPDATERESULT, updateresultMode); + MODECASE(STACKTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(STACKTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *having = psMetadataAlloc(); // HAVING clause + + PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "stackRun.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-select_filter", "stackRun.filter", "LIKE"); + + // these are used to build the HAVING restriction + PXOPT_COPY_S32(config->args, having, "-min_num", "num", ">="); + PXOPT_COPY_S32(config->args, having, "-min_filter", "num_filter", ">="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(where); + psFree(having); + return false; + } + + psString select = pxDataGet("staticskytool_definebyquery_select.sql"); + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + psFree(having); + return false; + } + + psString where1 = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&where1, "\nAND %s", whereClause); + psFree(whereClause); + } + + psString where2 = psStringCopy(""); + if (label) { + psStringAppend(&where2, "\nWHERE stackRun.label = '%s'", label); + } + + // Restriction on aggregated quantities using HAVING + { + psString havingClause = NULL; // HAVING string + if (psListLength(having->list)) { + havingClause = psDBGenerateWhereConditionSQL(having, NULL); + } + + if (min_new > 0) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + psStringAppend(&havingClause, + " (num_warp - num_stack >= %d OR (num_warp >= %d AND num_stack IS NULL))", + min_new, min_new); + } + if (isfinite(min_frac)) { + if (havingClause) { + psStringAppend(&havingClause, " AND"); + } + // Avoiding division by zero + psStringAppend(&havingClause, " (num_warp >= %f * num_stack OR num_stack IS NULL)", + (double)min_frac); + } + if (havingClause) { + psStringAppend(&select, " HAVING %s", havingClause); + psFree(havingClause); + } + } + psFree(having); + + if (!p_psDBRunQueryF(config->dbh, select, where1, where2)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + psFree(where1); + psFree(where2); + return false; + } + psFree(select); + psFree(where1); + psFree(where2); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("stacktool: no rows found"); + psFree(output); + psFree(where); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } + + psString insert = NULL; // Insertion query + if (randomLimit > 0) { + insert = pxDataGet("stacktool_definebyquery_insert_random_part1.sql"); + } else { + insert = pxDataGet("stacktool_definebyquery_insert.sql"); + } + if (!insert) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&insert, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (randomLimit > 0) { + psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql"); + if (!part2) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(insert); + return false; + } + psStringAppend(&insert, "%s", part2); + psFree(part2); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + // pull out the skycell_id, tess_id, filter + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup skycell_id"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup tess_id"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString filter = psMetadataLookupStr(&status, row, "filter"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup filter"); + psFree(output); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // create a new stackRun for this stack + stackRunRow *run = stackRunRowAlloc( + 0, // ID + "new", // state + workdir, + label, + data_group ? data_group : label, + dist_group, + reduction, + dvodb, + registered, + skycell_id, + tess_id, + filter, + note); + + if (!stackRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + psFree(insert); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // figure out the new stack_id + psS64 stack_id = psDBLastInsertID(config->dbh); + run->stack_id = stack_id; + + psArrayAdd(list, list->n, run); + psFree(run); + + // Create a suitable insertion query for this run + psString thisInsert = psStringCopy(insert); + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, stack_id); + psStringSubstitute(&thisInsert, idString, "@STACK_ID@"); + psFree(idString); + + // replace @FILTER@, @SKYCELL_ID@, @RANDOM_LIMIT@ + psStringSubstitute(&thisInsert, filter, "@FILTER@"); + psStringSubstitute(&thisInsert, skycell_id, "@SKYCELL_ID@"); + + if (randomLimit > 0) { + psString limString = NULL; + psStringAppend(&limString, "%d", randomLimit); + psStringSubstitute(&thisInsert, limString, "@RANDOM_LIMIT@"); + psFree(limString); + } + + // XXX this insert uses a select to generate the list of warp_ids for the stack, + // we have applied a set of criteria above (WHERE) to select the relevant warps + // this insert needs to use exactly the same restrictions (race condition is probably not critical) + // the insert below seems to only restrict matches to the skycell, tess, and filter + if (!p_psDBRunQuery(config->dbh, thisInsert)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(thisInsert); + psFree(insert); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(thisInsert); + +# if (0) + { + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("stacktool (definebyquery, insert): no rows found"); + psFree(output); + psFree(where); + return true; + } + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } +# endif + + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + if (!stackRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + +#ifdef notdef + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); +#endif + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE stackRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true); + + psFree(query); + psFree(where); + + return result; + +#ifdef notdef + // Hack-y work around to make stacktool more like the other tools, without breaking other stuff (hopefully). + + if ((state)&&(stack_id)) { + // set detRun.state to state + return setstackRunState(config, stack_id, state); + } + + if ((state)&&(label)) { + return setstackRunStateByLabel(config, label, state); + } + + psError(PS_ERR_UNKNOWN, false, "Required options not found."); + return false; +#endif +} + + +static bool inputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // XXX require at least a stack id (add better search options) + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_inputskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackInputSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool todoMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_tosum.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_stack, config->args, "-dtime_stack", false, false); + PXOPT_LOOKUP_F32(dtime_match_mean, config->args, "-dtime_match_mean", false, false); + PXOPT_LOOKUP_F32(dtime_match_stdev, config->args, "-dtime_match_stdev", false, false); + PXOPT_LOOKUP_F32(dtime_initial, config->args, "-dtime_initial", false, false); + PXOPT_LOOKUP_F32(dtime_reject, config->args, "-dtime_reject", false, false); + PXOPT_LOOKUP_F32(dtime_final, config->args, "-dtime_final", false, false); + PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_F32(match_mean, config->args, "-match_mean", false, false); + PXOPT_LOOKUP_F32(match_stdev, config->args, "-match_stdev", false, false); + PXOPT_LOOKUP_F32(match_rms, config->args, "-match_rms", false, false); + PXOPT_LOOKUP_F32(stamps_mean, config->args, "-stamps_mean", false, false); + PXOPT_LOOKUP_F32(stamps_stdev, config->args, "-stamps_stdev", false, false); + PXOPT_LOOKUP_S32(stamps_min, config->args, "-stamps_min", false, false); + PXOPT_LOOKUP_S32(reject_images, config->args, "-reject_images", false, false); + PXOPT_LOOKUP_F32(reject_pix_mean, config->args, "-reject_pix_mean", false, false); + PXOPT_LOOKUP_F32(reject_pix_stdev, config->args, "-reject_pix_stdev", false, false); + PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // XXX need to validate the stack_id here + // XXX instead of validiting it here we should just use forgein key + // constrants + if (!stackSumSkyfileInsert(config->dbh, + stack_id, + uri, + path_base, + bg, + bg_stdev, + dtime_stack, + dtime_match_mean, + dtime_match_stdev, + dtime_initial, + dtime_reject, + dtime_final, + dtime_phot, + dtime_script, + match_mean, + match_stdev, + match_rms, + stamps_mean, + stamps_stdev, + stamps_min, + reject_images, + reject_pix_mean, + reject_pix_stdev, + sources, + hostname, + good_frac, + fault, + quality + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (fault == 0) { + if (!setstackRunState(config, stack_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool resultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); + +// The following three selectors are incompatible with the sql so omit them +// PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); +// PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); +// PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_sumskyfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + + return true; +} + + +static bool updateresultState(pxConfig *config, psS64 stack_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, stack_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for stack_id %"PRId64, stack_id); + return false; + } + + return true; +} + +static bool updateresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 3; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + + ExportTable tables [] = { + {"stackRun", "stacktool_export_run.sql"}, + {"stackInputSkyfile", "stacktool_export_input_skyfile.sql"}, + {"stackSumSkyfile", "stacktool_export_sum_skyfile.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "stackRun")) { + if (!pxSetStateCleaned("stackRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + int numImportTables = 2; + + char tables[2] [80] = {"stackInputSkyfile", "stackSumSkyfile"}; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + +#ifdef notdef + fprintf (stderr, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + psMetadataItem *item = psMetadataLookup (input, "stackRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + stackRunRow *stackRun = stackRunObjectFromMetadata (entry->data.md); + stackRunInsertObject (config->dbh, stackRun); + + // fprintf (stdout, "---- stack run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + for (int i = 0; i < numImportTables; i++) { + psMetadataItem *item = psMetadataLookup (input, tables[i]); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + switch (i) { + case 0: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + stackInputSkyfileRow *stackInputSkyfile = stackInputSkyfileObjectFromMetadata (entry->data.md); + stackInputSkyfileInsertObject (config->dbh, stackInputSkyfile); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + + case 1: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + stackSumSkyfileRow *stackSumSkyfile = stackSumSkyfileObjectFromMetadata (entry->data.md); + stackSumSkyfileInsertObject (config->dbh, stackSumSkyfile); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + } + } + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/staticskytool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/staticskytool.h (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/staticskytool.h (revision 27840) @@ -0,0 +1,41 @@ +/* + * staticskytool.h + * + * Copyright (C) 2007,2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef STATICSKYTOOL_H +#define STATICSKYTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + STATICSKYTOOL_MODE_NONE = 0x0, + STATICSKYTOOL_MODE_DEFINEBYQUERY, + STATICSKYTOOL_MODE_UPDATERUN, + STATICSKYTOOL_MODE_INPUTS, + STATICSKYTOOL_MODE_TODO, + STATICSKYTOOL_MODE_ADDRESULT, + STATICSKYTOOL_MODE_RESULT, + STATICSKYTOOL_MODE_REVERT, + STATICSKYTOOL_MODE_UPDATERESULT, + STATICSKYTOOL_MODE_EXPORTRUN, + STATICSKYTOOL_MODE_IMPORTRUN +} staticskytoolMode; + +pxConfig *staticskytoolConfig(pxConfig *config, int argc, char **argv); + +#endif // STATICSKYTOOL_H Index: branches/simtest_nebulous_branches/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/staticskytoolConfig.c (revision 27840) +++ branches/simtest_nebulous_branches/ippTools/src/staticskytoolConfig.c (revision 27840) @@ -0,0 +1,173 @@ +/* + * staticskytoolConfig.c + * + * Copyright (C) 2007-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "staticskytool.h" + +pxConfig *staticskytoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ss_id", 0, "search by stack ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -inputs + psMetadata *inputsArgs = psMetadataAlloc(); + psMetadataAddS64(inputsArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -todo + psMetadata *todoArgs = psMetadataAlloc(); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(todoArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresult + psMetadata *addresultArgs = psMetadataAlloc(); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ss_id", 0, "define stack ID (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS32(addresultArgs, PS_LIST_TAIL, "-sources", 0, "number of sources", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-good_frac", 0, "define fraction of good pixels", NAN); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0) + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -result + psMetadata *resultArgs= psMetadataAlloc(); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revert + psMetadata *revertArgs= psMetadataAlloc(); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ss_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); + + // -updateresult + psMetadata *updateresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ss_id", 0, "define staticksky ID (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ss_id", 0, "export this staticsky ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "Define a new run", STATICSKYTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "Update a run", STATICSKYTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-inputs", "Get inputs", STATICSKYTOOL_MODE_INPUTS, inputsArgs); + PXOPT_ADD_MODE("-todo", "Get runs to do", STATICSKYTOOL_MODE_TODO, todoArgs); + PXOPT_ADD_MODE("-addresult", "Add result of run", STATICSKYTOOL_MODE_ADDRESULT, addresultsArgs); + PXOPT_ADD_MODE("-result", "Get result of run", STATICSKYTOOL_MODE_RESULT, resultArgs); + PXOPT_ADD_MODE("-revert", "Revert failed run", STATICSKYTOOL_MODE_REVERT, revertArgs); + PXOPT_ADD_MODE("-updateresult", "Update fault for run", STATICSKYTOOL_MODE_UPDATERESULT, updateresultArgs); + PXOPT_ADD_MODE("-exportrun", "Export run", STATICSKYTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "Import run", STATICSKYTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/simtest_nebulous_branches/ippTools/src/warptool.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/warptool.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/warptool.c (revision 27840) @@ -53,8 +53,12 @@ static bool tocleanedskyfileMode(pxConfig *config); static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); static bool tofullskyfileMode(pxConfig *config); static bool updateskyfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); @@ -102,8 +106,12 @@ MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); MODECASE(WARPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(WARPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(WARPTOOL_MODE_RUNSTATE, runstateMode); + MODECASE(WARPTOOL_MODE_LISTRUN, listrunMode); + MODECASE(WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); default: @@ -137,4 +145,7 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-note", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required (no default TESS) @@ -158,4 +169,6 @@ "dirty", // workdir_state label, + data_group ? data_group : label, + dist_group, dvodb, tess_id, @@ -163,5 +176,6 @@ end_stage, registered, - 0 // magicked + 0, // magicked + note ); if (!warpRun) { @@ -237,10 +251,10 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); // define using fake label if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -249,8 +263,11 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -266,5 +283,5 @@ psString query = pxDataGet("warptool_definebyquery.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -338,8 +355,11 @@ workdir ? workdir : row->workdir, label ? label : row->label, + data_group ? data_group: row->data_group, + dist_group ? dist_group: row->dist_group, dvodb ? dvodb : row->dvodb, tess_id ? tess_id : row->tess_id, reduction ? reduction : row->reduction, - end_stage ? end_stage : row->end_stage)) + end_stage ? end_stage : row->end_stage, + note)) { psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id); @@ -367,45 +387,33 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "warpRun.reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group","warpRun.data_group","=="); + PXOPT_COPY_STR(config->args, where, "-dist_group","warpRun.dist_group","=="); + pxAddLabelSearchArgs(config, where, "-label", "warpRun.label", "=="); + + PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpRun.registered", ">="); + PXOPT_COPY_TIME(config->args, where, "-registered_end", "warpRun.registered", "<"); + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + if (destreaked) { + psMetadataAddS64(where, PS_LIST_TAIL, "warpRun.magicked", PS_META_DUPLICATE_OK, ">", 0); + } + + if (!psListLength(where->list)) { psFree(where); - where = NULL; - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters (-state or -label) are required"); - psFree(where); - return false; - } - - if (state) { - // set warpRun.state to state - if (!pxwarpRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set chipRun.label to label - if (!pxwarpRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE warpRun JOIN warpSkyfile USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "warpRun", "warp_id", "warpSkyfile", true); + + psFree(query); psFree(where); - return true; -} - + return result; +} static bool expMode(pxConfig *config) @@ -423,5 +431,5 @@ psString query = pxDataGet("warptool_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -497,5 +505,5 @@ psString query = pxDataGet("warptool_imfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -571,5 +579,5 @@ psString query = pxDataGet("warptool_tooverlap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -680,5 +688,4 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyCellMap.tess_id", "=="); - // PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyCellMap.fault", "=="); @@ -687,5 +694,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -695,5 +702,5 @@ psString query = pxDataGet("warptool_revertoverlap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -827,5 +834,5 @@ psString query = pxDataGet("warptool_scmap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -900,11 +907,12 @@ psString query = pxDataGet("warptool_towarped.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereStr, "\n AND %s", whereClause); psFree(whereClause); } @@ -912,15 +920,29 @@ // treat limit == 0 as "no limit" + psString limitString = psStringCopy("\n"); if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); + // We apply the limit to both sides of the UNION + // to avoid slow queries and to the query itself + // to satisfy the user's requested limit + psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + } + +#define TWO_WHERES +#ifdef TWO_WHERES + if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr, limitString)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); return false; } +#else + if (!p_psDBRunQueryF(config->dbh, query, whereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } +#endif + psFree(limitString); + psFree(whereStr); psFree(query); @@ -981,5 +1003,5 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); // default values @@ -1044,5 +1066,5 @@ psString query = pxDataGet("warptool_finished_run_select.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); return false; } @@ -1120,5 +1142,5 @@ psString query = pxDataGet("warptool_finished_run_select.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); return false; } @@ -1196,4 +1218,12 @@ PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -1203,5 +1233,17 @@ psString query = pxDataGet("warptool_warped.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); return false; } @@ -1209,6 +1251,17 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } } psFree(where); @@ -1272,60 +1325,56 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); - // PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Delete product - int numDeleted; // Number deleted - { - psString query = pxDataGet("warptool_revertwarped_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("warptool_revertwarped_delete.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString query_updated = pxDataGet("warptool_revertwarped_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkyfiles", numDeleted); - psFree(where); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + // fix any faulted warpSkyfiles in data_state 'update' + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("warptool", PS_LOG_INFO, "Updated %d warpSkyfiles", numUpdated); return true; @@ -1414,5 +1463,5 @@ psString query = pxDataGet("warptool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1477,5 +1526,5 @@ psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1538,5 +1587,5 @@ psString query = pxDataGet("warptool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1626,7 +1675,20 @@ } - // note only updates if warpRun.state = run_state - // XXX note that we have removed this constraint for now - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) { + psString set_magicked_skyfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (!strcmp(data_state, "full")) { + // magicked is only an argument for for -tofullskyfile + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + if (magicked) { + psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked); + psStringAppend(&set_magicked_run, "\n, warpRun.magicked = %" PRId64, magicked); + } + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + // if magicked is currently nonzero set it to -1 + // Set warpRun.magicked when the first skyfile is cleaned + psStringAppend(&set_magicked_skyfile, "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1), warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)"); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); // rollback @@ -1638,7 +1700,8 @@ } psFree(query); + psFree(set_magicked_skyfile); query = pxDataGet("warptool_change_run_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) { // rollback if (!psDBRollback(config->dbh)) { @@ -1648,4 +1711,5 @@ return false; } + psFree(set_magicked_run); if (!psDBCommit(config->dbh)) { @@ -1667,4 +1731,8 @@ { return change_skyfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); } @@ -1677,12 +1745,30 @@ PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - - psString query = pxDataGet("warptool_updateskyfile.sql"); - - if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!state) { + psString query = pxDataGet("warptool_updateskyfile.sql"); + + if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + } + else { + if (strcmp(state,"error_cleaned") == 0) { + change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); + } + else if (strcmp(state, "error_scrubbed") == 0) { + change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); + } + else if (strcmp(state, "error_purged") == 0) { + change_skyfile_data_state(config,"error_purged","goto_purged"); + } + else { + psError(PS_ERR_UNKNOWN, false, "unhandled state given"); + return(false); + } + } return true; @@ -1709,4 +1795,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); @@ -1724,5 +1814,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1801,6 +1891,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "warpRun"); @@ -1865,2 +1962,234 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("warptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpRun.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "warpRun.dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpRun")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, warp_id DESC"); + } + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("warptool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , warpRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} Index: branches/simtest_nebulous_branches/ippTools/src/warptool.h =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/warptool.h (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/warptool.h (revision 27840) @@ -49,8 +49,12 @@ WARPTOOL_MODE_TOCLEANEDSKYFILE, WARPTOOL_MODE_TOPURGEDSKYFILE, + WARPTOOL_MODE_TOSCRUBBEDSKYFILE, WARPTOOL_MODE_TOFULLSKYFILE, WARPTOOL_MODE_UPDATESKYFILE, WARPTOOL_MODE_EXPORTRUN, - WARPTOOL_MODE_IMPORTRUN + WARPTOOL_MODE_IMPORTRUN, + WARPTOOL_MODE_RUNSTATE, + WARPTOOL_MODE_LISTRUN, + WARPTOOL_MODE_SETSKYFILETOUPDATE, } warptoolMode; Index: branches/simtest_nebulous_branches/ippTools/src/warptoolConfig.c =================================================================== --- branches/simtest_nebulous_branches/ippTools/src/warptoolConfig.c (revision 25061) +++ branches/simtest_nebulous_branches/ippTools/src/warptoolConfig.c (revision 27840) @@ -40,5 +40,5 @@ config->modules = pmConfigRead(&argc, argv, NULL); if (!config->modules) { - psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psError(psErrorCodeLast(), false, "Can't find site configuration"); psFree(config); return NULL; @@ -100,5 +100,7 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); @@ -117,4 +119,8 @@ psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); // -updaterun @@ -127,8 +133,17 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by warpRun reduction class", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by warpRun state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by warpRun label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by warpRun data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by warpRun dist_group", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_begin", 0, "search for runs by registration time (>=)", NULL); + psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_end", 0, "search for runs by registration time (<)", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -exp @@ -202,5 +217,5 @@ psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac", 0, "define %% of good pixels", NAN); - psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked", 0, "define if this skycell has been magicked", false); + psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-set_magicked", 0, "define if this skycell has been magicked", 0); psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); @@ -215,6 +230,45 @@ psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); + psMetadataAddS16(warpedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL); + // add magic related arguments + pxmagicAddArguments(warpedArgs); + // add arguments for spatial search + pxspaceAddArguments(warpedArgs); + + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); + psMetadataAddS16(listrunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL); + // add magic related arguments + pxmagicAddArguments(listrunArgs); + // add arguments for spatial search + pxspaceAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertwarped @@ -279,14 +333,21 @@ psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + // -toscrubbedskyfile + psMetadata *toscrubbedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + // -tofullskyfile psMetadata *tofullskyfileArgs = psMetadataAlloc(); psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); - - // -toupdateskyfile + psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-set_magicked", 0, "new value for magicked", 0); + + // -updateskyfile psMetadata *updateskyfileArgs = psMetadataAlloc(); psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-fault", 0, "new fault code", 0); + psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", 0); // -exportrun @@ -300,4 +361,22 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); +// psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -setskyfiletoupdate + psMetadata *setskyfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setskyfiletoupdateArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID (required)", 0); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(setskyfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "new value for warpRun.label", NULL); @@ -319,4 +398,6 @@ PXOPT_ADD_MODE("-advancerun", "", WARPTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-listrun", "", WARPTOOL_MODE_LISTRUN, listrunArgs); + PXOPT_ADD_MODE("-runstate", "", WARPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); PXOPT_ADD_MODE("-block", "set a label block", WARPTOOL_MODE_BLOCK, blockArgs); @@ -328,6 +409,8 @@ PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); + PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", WARPTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", WARPTOOL_MODE_IMPORTRUN, importrunArgs);